fix: retry a request when establishing the connection timed out

core/src/main/java/com/predic8/membrane/core/interceptor/HTTPClientInterceptor.java

@@ -19,6 +19,7 @@
 import com.predic8.membrane.core.exchange.Exchange;
 import com.predic8.membrane.core.http.MalformedHeaderException;
 import com.predic8.membrane.core.proxies.AbstractServiceProxy;
+import com.predic8.membrane.core.transport.http.ConnectTimeoutException;
 import com.predic8.membrane.core.transport.http.EOFWhileReadingLineException;
 import com.predic8.membrane.core.transport.http.HttpClient;
 import com.predic8.membrane.core.transport.http.ProtocolUpgradeDeniedException;
@@ -119,12 +120,21 @@ public Outcome handleRequest(Exchange exc) {
                     .buildAndSetResponse(exc);
             return ABORT;
         } catch (SocketTimeoutException e) {
-            // Details are logged further down in the HTTPClient
-            log.info("Target {} is not reachable.",exc.getDestinations());
+            // Name the phase: a connect timeout means the target never accepted the connection, a read
+            // timeout means it accepted but did not answer. Both used to be logged as
+            // "is not reachable.", which is also what a refused connection logs, so the three were
+            // indistinguishable in the log.
+            boolean whileConnecting = e instanceof ConnectTimeoutException;
+            var msg = "Target %s timed out %s.".formatted(getDestination(exc),
+                    whileConnecting ? "while the connection was being established, no request was sent"
+                                    : "while waiting for the response, the request had already been sent");
+            log.info("{} Reason: {}", msg, e.getMessage());
             internal(router.getConfiguration().isProduction(), getDisplayName())
                     .title("Gateway Timeout")
                     .status(504)
-                    .addSubSee("socket-timeout")
+                    .addSubSee(whileConnecting ? "connect-timeout" : "socket-timeout")
+                    .detail(msg)
+                    .stacktrace(false)
                     .buildAndSetResponse(exc);
             return ABORT;
         } catch (UnknownHostException e) {

core/src/main/java/com/predic8/membrane/core/transport/http/ConnectTimeoutException.java

@@ -0,0 +1,37 @@
+/* Copyright 2026 predic8 GmbH, www.predic8.com
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+package com.predic8.membrane.core.transport.http;
+
+import java.net.SocketTimeoutException;
+
+/**
+ * Indicates that establishing the connection to the target timed out, as opposed to the target
+ * accepting the connection but not answering in time. The JDK reports both as
+ * {@link SocketTimeoutException} and only distinguishes them by message text.
+ * <p>
+ * The distinction matters for retries: no byte of the request has been sent yet, so the target
+ * cannot have processed it and a retry is safe for any request method.
+ * <p>
+ * Extends {@link SocketTimeoutException} so that code catching the timeout in general keeps working.
+ */
+public class ConnectTimeoutException extends SocketTimeoutException {
+
+    private static final long serialVersionUID = 1L;
+
+    public ConnectTimeoutException(String message, Throwable cause) {
+        super(message);
+        initCause(cause);
+    }
+}

core/src/main/java/com/predic8/membrane/core/transport/http/Connection.java

@@ -102,25 +102,38 @@ public static Connection open(String host, int port, String localHost, SSLProvid
 			sniServername = null;
 		}
 
-		if (sslProvider != null) {
-			if (isNullOrEmpty(localHost))
-				con.socket = sslProvider.createSocket(host, port, connectTimeout, sniServername, applicationProtocols);
-			else
-				con.socket = sslProvider.createSocket(host, port, InetAddress.getByName(localHost), 0,
-						connectTimeout, sniServername, applicationProtocols);
-		} else {
-			if (isNullOrEmpty(localHost)) {
-				con.socket = new Socket();
+		// Everything up to here happens before the first byte of the request is written, so a timeout
+		// in this block means nothing was sent. ConnectTimeoutException carries that fact to the
+		// retry handling, which the JDK's undifferentiated SocketTimeoutException cannot.
+		try {
+			if (sslProvider != null) {
+				if (isNullOrEmpty(localHost))
+					con.socket = sslProvider.createSocket(host, port, connectTimeout, sniServername, applicationProtocols);
+				else
+					con.socket = sslProvider.createSocket(host, port, InetAddress.getByName(localHost), 0,
+							connectTimeout, sniServername, applicationProtocols);
 			} else {
-				con.socket = new Socket();
-				con.socket.bind(new InetSocketAddress(InetAddress.getByName(localHost), 0));
+				if (isNullOrEmpty(localHost)) {
+					con.socket = new Socket();
+				} else {
+					con.socket = new Socket();
+					con.socket.bind(new InetSocketAddress(InetAddress.getByName(localHost), 0));
+				}
+				con.socket.connect(new InetSocketAddress(host, port), connectTimeout);
 			}
-			con.socket.connect(new InetSocketAddress(host, port), connectTimeout);
-		}
 
-		if (proxy != null && origSSLProvider != null) {
-			con.doTunnelHandshake(proxy, con.socket, origHost, origPort);
-			con.socket = origSSLProvider.createSocket(con.socket, origHost, origPort, connectTimeout, origSniServername, applicationProtocols);
+			if (proxy != null && origSSLProvider != null) {
+				con.doTunnelHandshake(proxy, con.socket, origHost, origPort);
+				con.socket = origSSLProvider.createSocket(con.socket, origHost, origPort, connectTimeout, origSniServername, applicationProtocols);
+			}
+		} catch (SocketTimeoutException e) {
+			// The socket can already be open here: a timeout during the proxy handshake or the TLS
+			// wrapping happens after it was connected. Without closing it the descriptor leaks, and a
+			// retried connect attempt would leak one more.
+			ConnectTimeoutException timedOut = new ConnectTimeoutException(
+					"Connecting to %s:%d timed out after %dms.".formatted(host, port, connectTimeout), e);
+			closeSocket(con.socket, timedOut);
+			throw timedOut;
 		}
 
 		log.debug("Opened connection on localPort: {}", con.socket.getLocalPort());
@@ -129,6 +142,20 @@ public static Connection open(String host, int port, String localHost, SSLProvid
 		return con;
 	}
 
+	/**
+	 * Closes a socket that is being abandoned because opening the connection failed. A failure to close
+	 * is attached to the original exception rather than replacing it.
+	 */
+	private static void closeSocket(@Nullable Socket socket, Exception cause) {
+		if (socket == null)
+			return;
+		try {
+			socket.close();
+		} catch (IOException closeFailure) {
+			cause.addSuppressed(closeFailure);
+		}
+	}
+
 	private void setupStreams() throws IOException {
 		if (ByteStreamLogging.isLoggingEnabled()) {
 			String connectionName = chooseNewConnectionName();

core/src/main/java/com/predic8/membrane/core/transport/http/client/RetryHandler.java

@@ -43,6 +43,8 @@
  * <p>A retry is triggered for:</p>
  * <ul>
  *   <li>Connection/IO exceptions (timeout, refused, reset...)</li>
+ *   <li>A timeout while the connection was still being established (when
+ *       {@code retryOnConnectTimeout=true}), for any request method</li>
  *   <li>HTTP 408 Request Timeout</li>
  *   <li>HTTP 500, 502, 503, 504, 507 (when {@code failOverOn5XX=true})</li>
  * </ul>
@@ -74,6 +76,12 @@ public class RetryHandler {
      */
     private boolean failOverOn5XX = false;
 
+    /**
+     * Retry when establishing the connection timed out. Safe for any request method, because no part
+     * of the request was sent. Unlike a read timeout, this cannot have changed state on the server.
+     */
+    private boolean retryOnConnectTimeout = true;
+
     private static final Set<Integer> RETRYABLE_5XX = Set.of(500, 502, 503, 504, 507);
 
     /**
@@ -167,9 +175,17 @@ private boolean shouldAbortRetries(Exchange exc, Exception e, String dest, int a
             log.debug("Connection to {} refused.", dest);
             return !hasMultipleNodes(exc);
         }
-        // The socket read or connection took too long and exceeded the configured timeout.
-        // No data was received from the server in time.
-        // Causes: Server is overloaded, network latency or drop, TLS handshake took too long
+        // The connection was never established, so nothing was sent and no state was changed on the
+        // server. Retrying is safe for any method. Causes: dropped SYN, host unreachable, a TLS
+        // handshake that did not complete in time. Has to be checked before SocketTimeoutException,
+        // which it extends.
+        if (e instanceof ConnectTimeoutException) {
+            log.debug("Connection to {} timed out before it was established.", dest);
+            return !retryOnConnectTimeout;
+        }
+        // The socket read took too long and exceeded the configured timeout. No data was received
+        // from the server in time, but the request may already have been processed.
+        // Causes: Server is overloaded, network latency or drop
         if (e instanceof SocketTimeoutException) {
             log.debug("Connection to {} timed out.", dest);
             return !isIdempotent(exc.getRequest().getMethod()) || !hasMultipleNodes(exc);
@@ -303,6 +319,23 @@ public void setFailOverOn5XX(boolean failOverOn5XX) {
         this.failOverOn5XX = failOverOn5XX;
     }
 
+    public boolean isRetryOnConnectTimeout() {
+        return retryOnConnectTimeout;
+    }
+
+    /**
+     * @description If <code>true</code> retry when the connection to the target could not be
+     *              established within the connection timeout. No part of the request has been sent in
+     *              that case, so this applies to every request method, including POST and PATCH. A
+     *              timeout while reading the response is not covered by this and stays restricted to
+     *              idempotent methods. Set to <code>false</code> to fail fast instead.
+     * @default true
+     */
+    @MCAttribute
+    public void setRetryOnConnectTimeout(boolean retryOnConnectTimeout) {
+        this.retryOnConnectTimeout = retryOnConnectTimeout;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (o == null || getClass() != o.getClass()) return false;
@@ -311,7 +344,8 @@ public boolean equals(Object o) {
         return retries == that.retries &&
                delay == that.delay &&
                Double.compare(backoffMultiplier, that.backoffMultiplier) == 0 &&
-               Objects.equals(failOverOn5XX, that.failOverOn5XX);
+               Objects.equals(failOverOn5XX, that.failOverOn5XX) &&
+               retryOnConnectTimeout == that.retryOnConnectTimeout;
     }
 
     @Override
@@ -320,6 +354,7 @@ public int hashCode() {
         result = 31 * result + delay;
         result = 31 * result + Double.hashCode(backoffMultiplier);
         result = 31 * result + Boolean.hashCode(failOverOn5XX);
+        result = 31 * result + Boolean.hashCode(retryOnConnectTimeout);
         return result;
     }
 }

core/src/test/java/com/predic8/membrane/core/interceptor/HTTPClientInterceptorTest.java

@@ -20,6 +20,8 @@
 import com.predic8.membrane.core.openapi.serviceproxy.*;
 import com.predic8.membrane.core.proxies.*;
 import com.predic8.membrane.core.router.*;
+import com.predic8.membrane.core.transport.http.*;
+import com.predic8.membrane.core.transport.http.client.*;
 import com.predic8.membrane.core.util.*;
 import com.predic8.membrane.core.util.text.*;
 import com.predic8.membrane.core.util.text.SerializationUtil.*;
@@ -147,6 +149,80 @@ void computeCompletePathURLEncoded() throws Exception {
                 "%26%3F%C3%A4%C3%B6%C3%BC%21", Serialization.URL);
     }
 
+    /**
+     * A refused target, a target that never accepts and one that accepts but stays silent used to be
+     * logged and reported alike. They have to stay distinguishable by status, subSee and detail.
+     */
+    @Nested
+    class unreachableTarget {
+
+        @Test
+        void refusedTargetYields502() throws Exception {
+            int freePort;
+            try (var probe = new ServerSocket(0)) {
+                freePort = probe.getLocalPort();
+            } // closed again, so nothing listens on freePort
+
+            var exc = callTarget("http://localhost:" + freePort + "/", 0);
+
+            assertEquals(502, exc.getResponse().getStatusCode());
+            assertTrue(exc.getResponse().getBodyAsStringDecoded().contains("connect"));
+        }
+
+        @Test
+        void silentTargetYields504NamingTheReadPhase() throws Exception {
+            try (var silent = new ServerSocket(0)) {
+                // accepts the connection but never writes a response, so the client hits its read timeout
+                var exc = callTarget("http://localhost:" + silent.getLocalPort() + "/", 250);
+
+                assertEquals(504, exc.getResponse().getStatusCode());
+                var body = exc.getResponse().getBodyAsStringDecoded();
+                assertTrue(body.contains("socket-timeout"), body);
+                assertTrue(body.contains("waiting for the response"), body);
+            }
+        }
+
+        @Test
+        void connectTimeoutYields504NamingTheConnectPhase() throws Exception {
+            // A real dropped SYN is not reproducible here, so the client is made to report one
+            var hci = new HTTPClientInterceptor(new HttpClient() {
+                @Override
+                public void call(Exchange exc) throws Exception {
+                    throw new ConnectTimeoutException("Connecting to example.com:80 timed out after 10000ms.",
+                            new SocketTimeoutException("Connect timed out"));
+                }
+            });
+            hci.init(router);
+
+            var exc = get("http://example.com/").buildExchange();
+            exc.setProxy(new NullProxy());
+            exc.getDestinations().add("http://example.com/");
+
+            hci.handleRequest(exc);
+
+            assertEquals(504, exc.getResponse().getStatusCode());
+            var body = exc.getResponse().getBodyAsStringDecoded();
+            assertTrue(body.contains("connect-timeout"), body);
+            assertTrue(body.contains("no request was sent"), body);
+        }
+
+        private Exchange callTarget(String url, int soTimeout) throws Exception {
+            var config = new HttpClientConfiguration();
+            config.getConnection().setSoTimeout(soTimeout);
+            // a read timeout must not be retried, so the assertions see the first failure
+            config.getRetryHandler().setRetries(0);
+            hci.setHttpClientConfig(config);
+            hci.init(router);
+
+            var exc = get(url).buildExchange();
+            exc.setProxy(new NullProxy());
+            exc.getDestinations().add(url);
+
+            hci.handleRequest(exc);
+            return exc;
+        }
+    }
+
     @Nested
     class injection {
 

core/src/test/java/com/predic8/membrane/core/transport/http/ConnectionTest.java

@@ -19,6 +19,10 @@
 import com.predic8.membrane.core.router.*;
 import org.junit.jupiter.api.*;
 
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
 import static org.junit.jupiter.api.Assertions.*;
 
 public class ConnectionTest {
@@ -56,4 +60,42 @@ public void testIsSame() {
 		assertTrue(conLocalhost.isSame("localhost", 2000));
 		assertTrue(con127_0_0_1.isSame("127.0.0.1", 2000));
 	}
+
+	/**
+	 * A timeout while connecting has to be distinguishable from one while reading the response, because
+	 * only the former guarantees that nothing was sent. The JDK reports both as SocketTimeoutException.
+	 * A listening socket whose accept queue is full drops further SYNs, which is what makes the connect
+	 * time out here.
+	 */
+	@Test
+	void connectTimeoutSurfacesAsConnectTimeoutException() throws Exception {
+		List<Socket> queued = new ArrayList<>();
+		try (ServerSocket neverAccepting = new ServerSocket(0, 1)) {
+			int port = neverAccepting.getLocalPort();
+			fillAcceptQueue(port, queued);
+
+			assertThrows(ConnectTimeoutException.class,
+					() -> Connection.open("127.0.0.1", port, null, null, 200));
+		} finally {
+			for (Socket s : queued)
+				s.close();
+		}
+	}
+
+	/**
+	 * Connects until the accept queue of the never-accepting socket is full, from which point on the
+	 * kernel drops further SYNs rather than queueing them.
+	 */
+	private static void fillAcceptQueue(int port, List<Socket> queued) throws IOException {
+		for (int i = 0; i < 10; i++) {
+			Socket s = new Socket();
+			try {
+				s.connect(new InetSocketAddress("127.0.0.1", port), 200);
+				queued.add(s);
+			} catch (SocketTimeoutException e) {
+				s.close();
+				return;
+			}
+		}
+	}
 }