Fix DTLS half-open peer exhaustion and ClientHello RSA signing DoS

Hazel.UnitTests/Dtls/DtlsConnectionTests.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Net;
 using System.Net.Sockets;
+using System.Reflection;
 using System.Security.Cryptography;
 using System.Security.Cryptography.X509Certificates;
 using System.Threading;
@@ -85,6 +86,15 @@ private static X509Certificate2Collection GetCertificateForClient()
             return clientCertificates;
         }
 
+        private static int GetAvailableUdpPort()
+        {
+            using (Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp))
+            {
+                socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
+                return ((IPEndPoint)socket.LocalEndPoint).Port;
+            }
+        }
+
         protected DtlsConnectionListener CreateListener(int numWorkers, IPEndPoint endPoint, ILogger logger, IPMode ipMode = IPMode.IPv4)
         {
             DtlsConnectionListener listener = new DtlsConnectionListener(2, endPoint, logger, ipMode);
@@ -838,6 +848,191 @@ protected override void SendClientHello(bool isRetransmit)
             }
         }
 
+        private class ControlledClientHelloDtlsConnection : DtlsUnityConnection
+        {
+            public ControlledClientHelloDtlsConnection(ILogger logger, IPEndPoint remoteEndPoint)
+                : base(logger, remoteEndPoint)
+            {
+            }
+
+            public void ResendClientHelloForTest()
+            {
+                base.SendClientHello(true);
+            }
+        }
+
+        private sealed class CountingRsa : RSA
+        {
+            private RSA inner;
+
+            public int SignCount;
+
+            public CountingRsa(RSA inner)
+            {
+                this.inner = inner;
+            }
+
+            public override int KeySize
+            {
+                get => this.inner.KeySize;
+                set => this.inner.KeySize = value;
+            }
+
+            public override KeySizes[] LegalKeySizes => this.inner.LegalKeySizes;
+
+            public override byte[] Decrypt(byte[] data, RSAEncryptionPadding padding)
+            {
+                return this.inner.Decrypt(data, padding);
+            }
+
+            public override byte[] Encrypt(byte[] data, RSAEncryptionPadding padding)
+            {
+                return this.inner.Encrypt(data, padding);
+            }
+
+            public override RSAParameters ExportParameters(bool includePrivateParameters)
+            {
+                return this.inner.ExportParameters(includePrivateParameters);
+            }
+
+            public override void ImportParameters(RSAParameters parameters)
+            {
+                this.inner.ImportParameters(parameters);
+            }
+
+            public override byte[] SignHash(byte[] hash, HashAlgorithmName hashAlgorithm, RSASignaturePadding padding)
+            {
+                Interlocked.Increment(ref this.SignCount);
+                return this.inner.SignHash(hash, hashAlgorithm, padding);
+            }
+
+            public override bool VerifyHash(byte[] hash, byte[] signature, HashAlgorithmName hashAlgorithm, RSASignaturePadding padding)
+            {
+                return this.inner.VerifyHash(hash, signature, hashAlgorithm, padding);
+            }
+
+            protected override void Dispose(bool disposing)
+            {
+                if (disposing)
+                {
+                    this.inner?.Dispose();
+                    this.inner = null;
+                }
+
+                base.Dispose(disposing);
+            }
+        }
+
+        private class HalfOpenDtlsConnection : DtlsUnityConnection
+        {
+            public HalfOpenDtlsConnection(ILogger logger, IPEndPoint remoteEndPoint)
+                : base(logger, remoteEndPoint)
+            {
+            }
+
+            protected override void SendClientKeyExchangeFlight(bool isRetransmit)
+            {
+                // Stop after the cookie-verified ClientHello so the listener
+                // has a peer record but no application-level connection.
+            }
+        }
+
+        [TestMethod]
+        public void StaleHalfOpenPeerIsRemovedTest()
+        {
+            IPEndPoint listenerEndPoint = new IPEndPoint(IPAddress.Loopback, GetAvailableUdpPort());
+
+            using (DtlsConnectionListener listener = this.CreateListener(2, listenerEndPoint, new TestLogger("Server")))
+            using (HalfOpenDtlsConnection client = new HalfOpenDtlsConnection(new TestLogger("Client "), listenerEndPoint))
+            {
+                client.SetValidServerCertificates(GetCertificateForClient());
+
+                int connections = 0;
+                listener.NewConnection += (_) => Interlocked.Increment(ref connections);
+
+                listener.Start();
+                client.ConnectAsync();
+
+                Assert.IsTrue(
+                    SpinWait.SpinUntil(() => listener.PeerCount == 1, 5000),
+                    "The listener did not create a peer for the cookie-verified ClientHello.");
+                Assert.IsTrue(
+                    SpinWait.SpinUntil(() => listener.PeerCount == 0, 7000),
+                    "The listener did not evict the stale half-open peer.");
+                Assert.AreEqual(0, connections, "A half-open DTLS handshake must not create an application connection.");
+            }
+        }
+
+        [TestMethod]
+        public void StalePeerWithoutFinishedIsRemovedTest()
+        {
+            IPEndPoint listenerEndPoint = new IPEndPoint(IPAddress.Loopback, GetAvailableUdpPort());
+            IPEndPoint captureEndPoint = new IPEndPoint(IPAddress.Loopback, GetAvailableUdpPort());
+            int truncatedFlights = 0;
+
+            using (SocketCapture capture = new SocketCapture(captureEndPoint, listenerEndPoint, new TestLogger("Capture")))
+            using (DtlsConnectionListener listener = this.CreateListener(2, listenerEndPoint, new TestLogger("Server")))
+            using (DtlsUnityConnection client = this.CreateConnection(captureEndPoint, new TestLogger("Client ")))
+            using (Semaphore serverToClient = new Semaphore(0, int.MaxValue))
+            {
+                capture.SendToLocalSemaphore = serverToClient;
+                capture.PacketForRemoteTransform = packet =>
+                {
+                    if (!Record.Parse(out Record firstRecord, expectedProtocolVersion: null, packet))
+                    {
+                        return packet;
+                    }
+
+                    int secondRecordOffset = Record.Size + firstRecord.Length;
+                    if (firstRecord.ContentType != ContentType.Handshake || packet.Length <= secondRecordOffset
+                        || !Record.Parse(out Record secondRecord, expectedProtocolVersion: null, packet.Slice(secondRecordOffset)))
+                    {
+                        return packet;
+                    }
+
+                    int thirdRecordOffset = secondRecordOffset + Record.Size + secondRecord.Length;
+                    if (secondRecord.ContentType != ContentType.ChangeCipherSpec || packet.Length <= thirdRecordOffset
+                        || !Record.Parse(out Record thirdRecord, expectedProtocolVersion: null, packet.Slice(thirdRecordOffset))
+                        || thirdRecord.ContentType != ContentType.Handshake
+                        || thirdRecord.Epoch == 0)
+                    {
+                        return packet;
+                    }
+
+                    Interlocked.Increment(ref truncatedFlights);
+                    return packet.Slice(0, thirdRecordOffset);
+                };
+
+                int connections = 0;
+                listener.NewConnection += (_) => Interlocked.Increment(ref connections);
+
+                listener.Start();
+                client.ConnectAsync();
+
+                // Deliver the server flight in order so this test isolates the
+                // missing Finished state instead of exercising packet reordering.
+                capture.AssertPacketsToLocalCountEquals(1);
+                capture.ReleasePacketsToLocal(1);
+                capture.AssertPacketsToLocalCountEquals(3);
+                for (int i = 0; i < 3; ++i)
+                {
+                    capture.ReleasePacketsToLocal(1);
+                    Thread.Sleep(25);
+                }
+
+                Assert.IsTrue(
+                    SpinWait.SpinUntil(() => Volatile.Read(ref truncatedFlights) > 0, 5000),
+                    "The client did not send a ClientKeyExchange and ChangeCipherSpec flight.");
+                Assert.IsTrue(
+                    SpinWait.SpinUntil(() => listener.PeerCount == 1, 5000),
+                    "The listener did not retain the peer while waiting for Finished.");
+                Assert.IsTrue(
+                    SpinWait.SpinUntil(() => listener.PeerCount == 0, 7000),
+                    "The listener did not evict the peer after Finished was omitted.");
+                Assert.AreEqual(0, connections, "A DTLS handshake without Finished must not create an application connection.");
+            }
+        }
+
 
         private class MultipleClientKeyExchangeFlightDtlsConnection : DtlsUnityConnection
         {
@@ -882,6 +1077,40 @@ public void ConnectLikeAJerkTest()
             }
         }
 
+        [TestMethod]
+        public void ResentClientHelloReusesServerKeyExchangeSignatureTest()
+        {
+            IPEndPoint listenerEndPoint = new IPEndPoint(IPAddress.Loopback, GetAvailableUdpPort());
+            IPEndPoint captureEndPoint = new IPEndPoint(IPAddress.Loopback, GetAvailableUdpPort());
+
+            using (SocketCapture capture = new SocketCapture(captureEndPoint, listenerEndPoint, new TestLogger("Capture")))
+            using (DtlsConnectionListener listener = this.CreateListener(2, listenerEndPoint, new TestLogger("Server")))
+            using (ControlledClientHelloDtlsConnection client = new ControlledClientHelloDtlsConnection(new TestLogger("Client "), captureEndPoint))
+            using (Semaphore serverToClient = new Semaphore(0, int.MaxValue))
+            {
+                capture.SendToLocalSemaphore = serverToClient;
+                FieldInfo privateKeyField = typeof(DtlsConnectionListener).GetField("certificatePrivateKey", BindingFlags.Instance | BindingFlags.NonPublic);
+                CountingRsa countingRsa = new CountingRsa((RSA)privateKeyField.GetValue(listener));
+                privateKeyField.SetValue(listener, countingRsa);
+
+                client.SetValidServerCertificates(GetCertificateForClient());
+                listener.Start();
+                client.SetHandshakeResendTimeout(TimeSpan.FromSeconds(10));
+                client.ConnectAsync();
+
+                capture.AssertPacketsToLocalCountEquals(1);
+                capture.ReleasePacketsToLocal(1);
+
+                capture.AssertPacketsToLocalCountEquals(3);
+                Assert.AreEqual(1, countingRsa.SignCount);
+
+                client.ResendClientHelloForTest();
+                capture.AssertPacketsToLocalCountEquals(6);
+
+                Assert.AreEqual(1, countingRsa.SignCount, "ClientHello retransmission must reuse the signed ServerKeyExchange payload.");
+            }
+        }
+
         /// <summary>
         ///     Tests IPv4 resilience to multiple ClientKeyExchange packets.
         /// </summary>

Hazel.UnitTests/SocketCapture.cs

@@ -45,6 +45,7 @@ public class SocketCapture : IDisposable
 
         public Semaphore SendToLocalSemaphore = null;
         public Semaphore SendToRemoteSemaphore = null;
+        public Func<ByteSpan, ByteSpan> PacketForRemoteTransform = null;
 
         private CancellationTokenSource cancellationSource = new CancellationTokenSource();
         private readonly CancellationToken cancellationToken;
@@ -158,6 +159,7 @@ private void SendToRemoteLoop()
 
                     if (this.forRemote.TryTake(out var packet))
                     {
+                        packet = this.PacketForRemoteTransform?.Invoke(packet) ?? packet;
                         this.logger.WriteInfo($"Passed 1 packet of {packet.Length} bytes to remote");
                         this.captureSocket.SendTo(packet.GetUnderlyingArray(), packet.Offset, packet.Length, SocketFlags.None, this.remoteEndPoint);
                     }
@@ -303,4 +305,4 @@ internal string PacketsForLocalToString()
             return sb.ToString();
         }
     }
-}
+}