diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java index 68c5c3ca47ff93..e73fec2fa06c18 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.util; +import java.net.Inet4Address; +import java.net.Inet6Address; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; @@ -30,6 +32,7 @@ import org.apache.commons.net.util.SubnetUtils; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +41,7 @@ * answers membership queries. * * Accepts list of ip addresses, ip addreses in CIDR format and/or - * host addresses. + * host addresses. A CIDR entry is either an IPv4 or an IPv6 address range. */ public class MachineList { @@ -60,10 +63,49 @@ public InetAddress getByName (String host) throws UnknownHostException { } } + /** + * An IPv6 address range in CIDR format, which SubnetUtils does not support. + */ + private static final class Inet6Subnet { + private static final int MAX_PREFIX_LENGTH = 128; + + private final byte[] network; + private final int prefixLength; + + Inet6Subnet(String cidr) { + // Any other character is rejected, in particular a scope id because the prefix match + // would silently ignore it. + if (!cidr.matches("[0-9A-Fa-f:.]+/\\d{1,3}")) { + throw new IllegalArgumentException("Could not parse [" + cidr + "]"); + } + int slash = cidr.indexOf('/'); + InetAddress address = InetAddresses.forString(cidr.substring(0, slash)); + prefixLength = Integer.parseInt(cidr.substring(slash + 1)); + // An IPv4-mapped address is parsed as an Inet4Address and belongs in an IPv4 CIDR. + if (!(address instanceof Inet6Address) || prefixLength > MAX_PREFIX_LENGTH) { + throw new IllegalArgumentException("Could not parse [" + cidr + "]"); + } + network = address.getAddress(); + } + + boolean isInRange(byte[] address) { + int fullBytes = prefixLength / Byte.SIZE; + for (int i = 0; i < fullBytes; i++) { + if (address[i] != network[i]) { + return false; + } + } + int hostBits = Byte.SIZE - prefixLength % Byte.SIZE; + return hostBits == Byte.SIZE + || ((address[fullBytes] ^ network[fullBytes]) & 0xFF) >>> hostBits == 0; + } + } + private final boolean all; private final Set inetAddresses; private final Collection entries; private final List cidrAddresses; + private final List inet6CidrAddresses; private final InetAddressFactory addressFactory; /** @@ -101,17 +143,23 @@ public MachineList(Collection hostEntries, all = true; inetAddresses = null; cidrAddresses = null; + inet6CidrAddresses = null; } else { all = false; Set addrs = new HashSet<>(); List cidrs = new LinkedList(); + List inet6Cidrs = new ArrayList<>(); for (String hostEntry : hostEntries) { //ip address range if (hostEntry.indexOf("/") > -1) { try { - SubnetUtils subnet = new SubnetUtils(hostEntry); - subnet.setInclusiveHostCount(true); - cidrs.add(subnet.getInfo()); + if (hostEntry.indexOf(':') > -1) { + inet6Cidrs.add(new Inet6Subnet(hostEntry)); + } else { + SubnetUtils subnet = new SubnetUtils(hostEntry); + subnet.setInclusiveHostCount(true); + cidrs.add(subnet.getInfo()); + } } catch (IllegalArgumentException e) { LOG.warn("Invalid CIDR syntax : " + hostEntry); throw e; @@ -126,11 +174,13 @@ public MachineList(Collection hostEntries, } inetAddresses = (addrs.size() > 0) ? addrs : null; cidrAddresses = (cidrs.size() > 0) ? cidrs : null; + inet6CidrAddresses = (inet6Cidrs.size() > 0) ? inet6Cidrs : null; } } else { all = false; inetAddresses = null; cidrAddresses = null; + inet6CidrAddresses = null; entries = Collections.emptyList(); } } @@ -175,7 +225,8 @@ public boolean includes(InetAddress address) { return true; } // iterate through the ip ranges for inclusion - if (cidrAddresses != null) { + // an address never matches an ip range of the other address family + if (cidrAddresses != null && address instanceof Inet4Address) { String ipAddress = address.getHostAddress(); for(SubnetUtils.SubnetInfo cidrAddress : cidrAddresses) { if(cidrAddress.isInRange(ipAddress)) { @@ -183,6 +234,14 @@ public boolean includes(InetAddress address) { } } } + if (inet6CidrAddresses != null && address instanceof Inet6Address) { + byte[] ipAddress = address.getAddress(); + for (Inet6Subnet cidrAddress : inet6CidrAddresses) { + if (cidrAddress.isInRange(ipAddress)) { + return true; + } + } + } return false; } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java index c14972056cf6ba..16646e17476078 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.security.authorize; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -338,6 +340,39 @@ public void testBlockedMachineList() throws UnknownHostException { } } + @Test + public void testBlockedMachineListWithIPv6Client() throws UnknownHostException { + UserGroupInformation drwho = + UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM", + new String[] {"group1", "group2"}); + InetAddress ipv6Client = InetAddress.getByName("2001:db8::1"); + + ServiceAuthorizationManager serviceAuthorizationManager = + new ServiceAuthorizationManager(); + Configuration conf = new Configuration(); + + // a blocked MachineList with only IPv4 ranges does not block an IPv6 client + conf.set(BLOCKED_HOST_CONFIG, IP_RANGE); + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + assertThatCode(() -> serviceAuthorizationManager.authorize(drwho, + TestProtocol.class, conf, ipv6Client)).doesNotThrowAnyException(); + + // a blocked MachineList with a matching IPv6 range blocks the IPv6 client + conf.set(BLOCKED_HOST_CONFIG, IP_RANGE + ",2001:db8::/32"); + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + assertThatThrownBy(() -> serviceAuthorizationManager.authorize(drwho, + TestProtocol.class, conf, ipv6Client)) + .isInstanceOf(AuthorizationException.class) + .hasMessageContaining("Host 2001:db8:0:0:0:0:0:1 is not authorized"); + assertThatThrownBy(() -> serviceAuthorizationManager.authorize(drwho, + TestProtocol.class, conf, InetAddress.getByName("10.222.0.0"))) + .isInstanceOf(AuthorizationException.class); + // an IPv6 client outside of the blocked range is not blocked + assertThatCode(() -> serviceAuthorizationManager.authorize(drwho, + TestProtocol.class, conf, InetAddress.getByName("2001:db9::1"))) + .doesNotThrowAnyException(); + } + @Test public void testDefaultBlockedMachineList() throws UnknownHostException { UserGroupInformation drwho = diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java index f5c38937895960..90d2e64d6101a2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java @@ -17,12 +17,15 @@ */ package org.apache.hadoop.util; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import java.net.Inet6Address; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Collection; @@ -42,6 +45,9 @@ public class TestMachineList { private static String INVALID_CIDR = "10.241/24"; private static String IP_CIDR_LIST = "10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24"; + private static final String IPV6_CIDR_LIST = "2001:db8::/32,fd00:1:2:3::/64"; + private static final String IPV4_IPV6_LIST = + "10.222.0.0/16,2001:db8::/32,10.119.103.110,fd00::5"; private static String HOST_LIST = "host1,host4"; private static String HOSTNAME_IP_CIDR_LIST = "host1,10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24,host4,"; @@ -310,4 +316,151 @@ public void testGetCollection() { assertTrue(col.contains(item)); } } + + @Test + public void testIPv4CIDREdgeCases() { + MachineList ml = new MachineList("10.1.2.3/32,10.1.3.2/31", new TestAddressFactory()); + + assertThat(ml.includes("10.1.2.2")).isFalse(); + assertThat(ml.includes("10.1.2.3")).isTrue(); + assertThat(ml.includes("10.1.2.4")).isFalse(); + + assertThat(ml.includes("10.1.3.1")).isFalse(); + assertThat(ml.includes("10.1.3.2")).isTrue(); + assertThat(ml.includes("10.1.3.3")).isTrue(); + assertThat(ml.includes("10.1.3.4")).isFalse(); + + ml = new MachineList("0.0.0.0/0", new TestAddressFactory()); + assertThat(ml.includes("1.2.3.4")).isTrue(); + assertThat(ml.includes("255.255.255.255")).isTrue(); + assertThat(ml.includes("2001:db8::1")).isFalse(); + + assertThatThrownBy(() -> new MachineList("10.0.0.0/33", new TestAddressFactory())) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testIPv6List() { + MachineList ml = new MachineList("2001:db8::1,fd00::5", new TestAddressFactory()); + + assertThat(ml.includes("2001:db8::1")).isTrue(); + assertThat(ml.includes("2001:db8:0:0:0:0:0:1")).isTrue(); + assertThat(ml.includes("fd00::5")).isTrue(); + assertThat(ml.includes("2001:db8::2")).isFalse(); + assertThat(ml.includes("10.119.103.112")).isFalse(); + } + + @Test + public void testIPv6CIDRs() { + MachineList ml = new MachineList(IPV6_CIDR_LIST, new TestAddressFactory()); + + assertThat(ml.includes("2001:db7:ffff:ffff:ffff:ffff:ffff:ffff")).isFalse(); + assertThat(ml.includes("2001:db8::")).isTrue(); + assertThat(ml.includes("2001:db8::1")).isTrue(); + assertThat(ml.includes("2001:db8:0:0:0:0:0:1")).isTrue(); + assertThat(ml.includes("2001:db8:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue(); + assertThat(ml.includes("2001:db9::")).isFalse(); + + assertThat(ml.includes("fd00:1:2:2:ffff:ffff:ffff:ffff")).isFalse(); + assertThat(ml.includes("fd00:1:2:3::")).isTrue(); + assertThat(ml.includes("fd00:1:2:3:ffff:ffff:ffff:ffff")).isTrue(); + assertThat(ml.includes("fd00:1:2:4::")).isFalse(); + + // host bits in the entry are ignored, as for an IPv4 CIDR + ml = new MachineList("2001:db8::1/32", new TestAddressFactory()); + assertThat(ml.includes("2001:db8:ffff::2")).isTrue(); + assertThat(ml.includes("2001:db9::1")).isFalse(); + } + + @Test + public void testIPv6CIDRPrefixNotOnByteBoundary() { + MachineList ml = + new MachineList("2001:db8:8000::/33,fd00:0:0:8::/61", new TestAddressFactory()); + + assertThat(ml.includes("2001:db8:7fff:ffff:ffff:ffff:ffff:ffff")).isFalse(); + assertThat(ml.includes("2001:db8:8000::")).isTrue(); + assertThat(ml.includes("2001:db8:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue(); + assertThat(ml.includes("2001:db9::")).isFalse(); + + assertThat(ml.includes("fd00:0:0:7:ffff:ffff:ffff:ffff")).isFalse(); + assertThat(ml.includes("fd00:0:0:8::")).isTrue(); + assertThat(ml.includes("fd00:0:0:f:ffff:ffff:ffff:ffff")).isTrue(); + assertThat(ml.includes("fd00:0:0:10::")).isFalse(); + } + + @Test + public void testIPv6CIDRWithShortestAndLongestPrefix() { + MachineList ml = new MachineList("2001:db8::1/128", new TestAddressFactory()); + assertThat(ml.includes("2001:db8::")).isFalse(); + assertThat(ml.includes("2001:db8::1")).isTrue(); + assertThat(ml.includes("2001:db8::2")).isFalse(); + assertThat(ml.includes("2001:db8::8001")).isFalse(); + + ml = new MachineList("::/0", new TestAddressFactory()); + assertThat(ml.includes("::")).isTrue(); + assertThat(ml.includes("2001:db8::1")).isTrue(); + assertThat(ml.includes("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue(); + assertThat(ml.includes("10.119.103.112")).isFalse(); + } + + @Test + public void testIPv4AndIPv6CIDRs() { + MachineList ml = new MachineList(IPV4_IPV6_LIST, new TestAddressFactory()); + + assertThat(ml.includes("10.222.255.255")).isTrue(); + assertThat(ml.includes("10.223.0.0")).isFalse(); + assertThat(ml.includes("10.119.103.110")).isTrue(); + assertThat(ml.includes("2001:db8:ffff::1")).isTrue(); + assertThat(ml.includes("2001:db9::1")).isFalse(); + assertThat(ml.includes("fd00::5")).isTrue(); + assertThat(ml.includes("fd00::6")).isFalse(); + } + + @Test + public void testCIDRsOfOtherAddressFamily() throws UnknownHostException { + // an IPv6 address is not in an IPv4 range, the lookup must not throw + MachineList ml = new MachineList(CIDR_LIST, new TestAddressFactory()); + assertThat(ml.includes("2001:db8::1")).isFalse(); + assertThat(ml.includes(InetAddress.getByName("2001:db8::1"))).isFalse(); + assertThat(ml.includes("::10.222.0.1")).isFalse(); + + ml = new MachineList(IPV6_CIDR_LIST, new TestAddressFactory()); + assertThat(ml.includes("10.222.0.1")).isFalse(); + // 32.1.13.184 has the same leading bytes as 2001:db8:: + assertThat(ml.includes(InetAddress.getByName("32.1.13.184"))).isFalse(); + } + + @Test + public void testIPv4MappedIPv6Address() { + // the JDK turns an IPv4-mapped IPv6 address into an IPv4 address + MachineList ml = new MachineList(CIDR_LIST, new TestAddressFactory()); + assertThat(ml.includes("::ffff:10.222.0.1")).isTrue(); + assertThat(ml.includes("::ffff:10.223.0.0")).isFalse(); + + ml = new MachineList("::/0", new TestAddressFactory()); + assertThat(ml.includes("::ffff:10.222.0.1")).isFalse(); + } + + @Test + public void testIPv6CIDRIgnoresScopeIdOfAddress() throws UnknownHostException { + MachineList ml = new MachineList("fe80::/10", new TestAddressFactory()); + byte[] linkLocal = InetAddress.getByName("fe80::1").getAddress(); + byte[] siteLocal = InetAddress.getByName("fec0::1").getAddress(); + + assertThat(ml.includes(Inet6Address.getByAddress(null, linkLocal, 1))).isTrue(); + assertThat(ml.includes(Inet6Address.getByAddress(null, siteLocal, 1))).isFalse(); + } + + @Test + public void testInvalidIPv6CIDR() { + String[] invalidCidrs = {"2001:db8::/129", "2001:db8::/1000", "2001:db8::/-1", + "2001:db8::/+32", "2001:db8::/x", "2001:db8::/", "2001:db8::/32/64", "2001:db8::g/32", + ":/32", "[2001:db8::]/32", "fe80::%1/64", "fe80::%/64", "2001:db8::\uff11/128", + "::ffff:10.222.0.0/112"}; + for (String invalidCidr : invalidCidrs) { + assertThatThrownBy(() -> new MachineList(invalidCidr, new TestAddressFactory())) + .as(invalidCidr) + .isInstanceOf(IllegalArgumentException.class); + } + } }