HBASE-30355 setupMiniKdc port-conflict retry never triggers because Kerby throws KrbException, not BindException

The retry-on-port-conflict block in setupMiniKdc only catches
java.net.BindException, but the Kerby-backed MiniKdc wraps the bind failure in
a KrbException with no BindException in its cause chain - the conflict is only
visible via the "Address already in use" message. The retry was therefore dead
code and the first port collision failed the test.

Recognise the bind conflict regardless of wrapper type: catch Exception, and
before retrying, test a predicate that walks the whole cause chain and matches
the "Address already in use" message; rethrow anything else so a genuine KDC
misconfig is not masked. We avoid catch (BindException | KrbException) because
the message (not the type) is the discriminator and to avoid importing kerby
types here (see HBASE-29117).

Applied to both TestLogLevel.setupMiniKdc (hbase-http) and
HBaseTestingUtil.setupMiniKdc (hbase-server).

Tests:
- testKdcBindConflictSurfacesAsKrbException: deterministic reproduction
  occupying the KDC port on TCP+UDP; asserts the failure is a KrbException the
  predicate recognises.
- testIsBindExceptionRecognizesKerbyWrappedBindFailure: unit test for the
  predicate (wrapped BindException, message-only, and negative case).

Author: mnpoonia

hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java

@@ -21,12 +21,16 @@
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.io.File;
 import java.io.IOException;
 import java.net.BindException;
+import java.net.DatagramSocket;
+import java.net.InetAddress;
+import java.net.ServerSocket;
 import java.net.SocketException;
 import java.net.URI;
 import java.security.PrivilegedExceptionAction;
@@ -57,6 +61,7 @@
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.kerby.kerberos.kerb.KrbException;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
@@ -153,20 +158,95 @@ static private MiniKdc setupMiniKdc() throws Exception {
         dir = new File(HTU.getDataTestDir("kdc").toUri().getPath());
         kdc = new MiniKdc(conf, dir);
         kdc.start();
-      } catch (BindException e) {
+      } catch (Exception e) {
+        // Catch Exception, not BindException/KrbException: Kerby wraps the bind failure in a
+        // KrbException (and the cause chain carries no BindException), so the port conflict is only
+        // recognisable by message. We also avoid importing kerby types here (see HBASE-29117).
+        if (!isBindException(e)) {
+          throw e; // not a port conflict, do not mask the real failure behind a retry
+        }
         FileUtils.deleteDirectory(dir); // clean directory
         numTries++;
         if (numTries == 3) {
           log.error("Failed setting up MiniKDC. Tried " + numTries + " times.");
           throw e;
         }
-        log.error("BindException encountered when setting up MiniKdc. Trying again.");
+        log.error("Bind conflict encountered when setting up MiniKdc, retrying (attempt " + numTries
+          + ").");
         bindException = true;
       }
     } while (bindException);
     return kdc;
   }
 
+  /**
+   * The Kerby-backed {@link MiniKdc} wraps a failure to bind the KDC port in a
+   * {@code org.apache.kerby...KrbException} rather than surfacing a {@link BindException} directly,
+   * so we inspect the whole cause chain plus the message to recognise a port conflict.
+   */
+  static boolean isBindException(Throwable t) {
+    for (Throwable cause = t; cause != null; cause = cause.getCause()) {
+      if (cause instanceof BindException) {
+        return true;
+      }
+      String msg = cause.getMessage();
+      if (msg != null && msg.contains("Address already in use")) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Reproduces the port-conflict flake deterministically: occupy the KDC port on both TCP and UDP,
+   * pin MiniKdc to it, and show the resulting failure is a Kerby {@code KrbException}, not a
+   * {@link BindException}. This is why the {@code catch (BindException)} retry in
+   * {@link #setupMiniKdc()} never fired for the reported failure.
+   */
+  @Test
+  public void testKdcBindConflictSurfacesAsKrbException() throws Exception {
+    int port;
+    try (ServerSocket probe = new ServerSocket(0, 1, InetAddress.getByName(LOCALHOST))) {
+      port = probe.getLocalPort();
+    }
+    try (ServerSocket tcp = new ServerSocket(port, 1, InetAddress.getByName(LOCALHOST));
+      DatagramSocket udp = new DatagramSocket(port, InetAddress.getByName(LOCALHOST))) {
+      Properties conf = MiniKdc.createConf();
+      conf.put(MiniKdc.DEBUG, true);
+      conf.setProperty(MiniKdc.KDC_BIND_ADDRESS, LOCALHOST);
+      conf.setProperty(MiniKdc.KDC_PORT, Integer.toString(port));
+      File dir = new File(HTU.getDataTestDir("kdc-conflict").toUri().getPath());
+      MiniKdc conflictingKdc = new MiniKdc(conf, dir);
+
+      Exception thrown = assertThrows(Exception.class, conflictingKdc::start);
+
+      assertFalse(thrown instanceof BindException,
+        "Kerby wraps the bind failure in a KrbException, so it is not a BindException: " + thrown);
+      assertTrue(isBindException(thrown),
+        "expected a recognisable bind-conflict failure, got: " + thrown);
+    }
+  }
+
+  /**
+   * Deterministic regression test for the retry predicate: a Kerby {@code KrbException} caused by a
+   * port conflict must be recognised as a bind failure, whether the {@link BindException} is
+   * preserved in the cause chain or only reflected in the message. Unrelated failures must not be.
+   */
+  @Test
+  public void testIsBindExceptionRecognizesKerbyWrappedBindFailure() {
+    assertTrue(
+      isBindException(
+        new KrbException("Failed to start DefaultKrbServer. Address already in use (Bind failed)",
+          new BindException("Address already in use (Bind failed)"))),
+      "KrbException wrapping a BindException should be recognised");
+    assertTrue(
+      isBindException(
+        new KrbException("Failed to start DefaultKrbServer. Address already in use (Bind failed)")),
+      "KrbException whose message reports the bind conflict should be recognised");
+    assertFalse(isBindException(new RuntimeException("boom")),
+      "an unrelated failure must not be treated as a retryable bind conflict");
+  }
+
   static private void setupSSL(File base) throws Exception {
     clientConf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());
     clientConf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");

hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java

@@ -3731,21 +3731,46 @@ public MiniKdc setupMiniKdc(File keytabFile) throws Exception {
         dir = new File(getDataTestDir("kdc").toUri().getPath());
         kdc = new MiniKdc(conf, dir);
         kdc.start();
-      } catch (BindException e) {
+      } catch (Exception e) {
+        // Catch Exception, not BindException/KrbException: Kerby wraps the bind failure in a
+        // KrbException (and the cause chain carries no BindException), so the port conflict is only
+        // recognisable by message. We also avoid importing kerby types here (see HBASE-29117).
+        if (!isBindException(e)) {
+          throw e; // not a port conflict, do not mask the real failure behind a retry
+        }
         FileUtils.deleteDirectory(dir); // clean directory
         numTries++;
         if (numTries == 3) {
           LOG.error("Failed setting up MiniKDC. Tried " + numTries + " times.");
           throw e;
         }
-        LOG.error("BindException encountered when setting up MiniKdc. Trying again.");
+        LOG.error("Bind conflict encountered when setting up MiniKdc, retrying (attempt " + numTries
+          + ").");
         bindException = true;
       }
     } while (bindException);
     HBaseKerberosUtils.setKeytabFileForTesting(keytabFile.getAbsolutePath());
     return kdc;
   }
 
+  /**
+   * The Kerby-backed {@link MiniKdc} wraps a failure to bind the KDC port in a
+   * {@code org.apache.kerby...KrbException} rather than surfacing a {@link BindException} directly,
+   * so we inspect the whole cause chain plus the message to recognise a port conflict.
+   */
+  static boolean isBindException(Throwable t) {
+    for (Throwable cause = t; cause != null; cause = cause.getCause()) {
+      if (cause instanceof BindException) {
+        return true;
+      }
+      String msg = cause.getMessage();
+      if (msg != null && msg.contains("Address already in use")) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   public int getNumHFiles(final TableName tableName, final byte[] family) {
     int numHFiles = 0;
     for (RegionServerThread regionServerThread : getMiniHBaseCluster().getRegionServerThreads()) {