Skip to content

HADOOP-19962. Support IPv6 CIDR rules in MachineList #8812

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
smengcl merged 3 commits into from
Oct 13, 2026
Merged

HADOOP-19962. Support IPv6 CIDR rules in MachineList #8812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7e544f3
HADOOP-19962. Support IPv6 CIDR rules in MachineList
smengcl Oct 11, 2026
1bf4cd7
HADOOP-19962. Add service authorization test for IPv6 clients and blo…
smengcl Oct 12, 2026
7219b86
HADOOP-19962. Validate the IPv6 CIDR format first and parse the prefi…
smengcl Oct 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 64 additions & 5 deletions hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
View file
Open in desktop
Comment thread
smengcl marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
*/
package org.apache.hadoop.util;

import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
Expand All @@ -30,6 +32,7 @@
import org.apache.commons.net.util.SubnetUtils;

import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -38,7 +41,7 @@
* answers membership queries.
*
* Accepts list of ip addresses, ip addreses in CIDR format and/or
* host addresses.
* host addresses. A CIDR entry is either an IPv4 or an IPv6 address range.
*/

public class MachineList {
Expand All @@ -60,10 +63,49 @@ public InetAddress getByName (String host) throws UnknownHostException {
}
}

/**
* An IPv6 address range in CIDR format, which SubnetUtils does not support.
*/
private static final class Inet6Subnet {
private static final int MAX_PREFIX_LENGTH = 128;

private final byte[] network;
private final int prefixLength;

Inet6Subnet(String cidr) {
// Any other character is rejected, in particular a scope id because the prefix match
// would silently ignore it.
if (!cidr.matches("[0-9A-Fa-f:.]+/\\d{1,3}")) {
throw new IllegalArgumentException("Could not parse [" + cidr + "]");
}
int slash = cidr.indexOf('/');
InetAddress address = InetAddresses.forString(cidr.substring(0, slash));
prefixLength = Integer.parseInt(cidr.substring(slash + 1));
// An IPv4-mapped address is parsed as an Inet4Address and belongs in an IPv4 CIDR.
if (!(address instanceof Inet6Address) || prefixLength > MAX_PREFIX_LENGTH) {
throw new IllegalArgumentException("Could not parse [" + cidr + "]");
}
network = address.getAddress();
}

boolean isInRange(byte[] address) {
int fullBytes = prefixLength / Byte.SIZE;
for (int i = 0; i < fullBytes; i++) {
if (address[i] != network[i]) {
return false;
}
}
int hostBits = Byte.SIZE - prefixLength % Byte.SIZE;
return hostBits == Byte.SIZE
|| ((address[fullBytes] ^ network[fullBytes]) & 0xFF) >>> hostBits == 0;
}
}

private final boolean all;
private final Set<InetAddress> inetAddresses;
private final Collection<String> entries;
private final List<SubnetUtils.SubnetInfo> cidrAddresses;
private final List<Inet6Subnet> inet6CidrAddresses;
private final InetAddressFactory addressFactory;

/**
Expand Down Expand Up @@ -101,17 +143,23 @@ public MachineList(Collection<String> hostEntries,
all = true;
inetAddresses = null;
cidrAddresses = null;
inet6CidrAddresses = null;
} else {
all = false;
Set<InetAddress> addrs = new HashSet<>();
List<SubnetUtils.SubnetInfo> cidrs = new LinkedList<SubnetUtils.SubnetInfo>();
List<Inet6Subnet> inet6Cidrs = new ArrayList<>();
for (String hostEntry : hostEntries) {
//ip address range
if (hostEntry.indexOf("/") > -1) {
try {
SubnetUtils subnet = new SubnetUtils(hostEntry);
subnet.setInclusiveHostCount(true);
cidrs.add(subnet.getInfo());
if (hostEntry.indexOf(':') > -1) {
inet6Cidrs.add(new Inet6Subnet(hostEntry));
} else {
SubnetUtils subnet = new SubnetUtils(hostEntry);
subnet.setInclusiveHostCount(true);
cidrs.add(subnet.getInfo());
}
} catch (IllegalArgumentException e) {
LOG.warn("Invalid CIDR syntax : " + hostEntry);
throw e;
Expand All @@ -126,11 +174,13 @@ public MachineList(Collection<String> hostEntries,
}
inetAddresses = (addrs.size() > 0) ? addrs : null;
cidrAddresses = (cidrs.size() > 0) ? cidrs : null;
inet6CidrAddresses = (inet6Cidrs.size() > 0) ? inet6Cidrs : null;
}
} else {
all = false;
inetAddresses = null;
cidrAddresses = null;
inet6CidrAddresses = null;
entries = Collections.emptyList();
}
}
Expand Down Expand Up @@ -175,14 +225,23 @@ public boolean includes(InetAddress address) {
return true;
}
// iterate through the ip ranges for inclusion
if (cidrAddresses != null) {
// an address never matches an ip range of the other address family
if (cidrAddresses != null && address instanceof Inet4Address) {
String ipAddress = address.getHostAddress();
for(SubnetUtils.SubnetInfo cidrAddress : cidrAddresses) {
if(cidrAddress.isInRange(ipAddress)) {
return true;
}
}
}
if (inet6CidrAddresses != null && address instanceof Inet6Address) {
byte[] ipAddress = address.getAddress();
for (Inet6Subnet cidrAddress : inet6CidrAddresses) {
if (cidrAddress.isInRange(ipAddress)) {
return true;
}
}
}
return false;
}
/**
Expand Down
35 changes: 35 additions & 0 deletions ...p-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
*/
package org.apache.hadoop.security.authorize;

import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;

Expand Down Expand Up @@ -338,6 +340,39 @@ public void testBlockedMachineList() throws UnknownHostException {
}
}

@Test
public void testBlockedMachineListWithIPv6Client() throws UnknownHostException {
UserGroupInformation drwho =
UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
new String[] {"group1", "group2"});
InetAddress ipv6Client = InetAddress.getByName("2001:db8::1");

ServiceAuthorizationManager serviceAuthorizationManager =
new ServiceAuthorizationManager();
Configuration conf = new Configuration();

// a blocked MachineList with only IPv4 ranges does not block an IPv6 client
conf.set(BLOCKED_HOST_CONFIG, IP_RANGE);
serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
assertThatCode(() -> serviceAuthorizationManager.authorize(drwho,
TestProtocol.class, conf, ipv6Client)).doesNotThrowAnyException();

// a blocked MachineList with a matching IPv6 range blocks the IPv6 client
conf.set(BLOCKED_HOST_CONFIG, IP_RANGE + ",2001:db8::/32");
serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
assertThatThrownBy(() -> serviceAuthorizationManager.authorize(drwho,
TestProtocol.class, conf, ipv6Client))
.isInstanceOf(AuthorizationException.class)
.hasMessageContaining("Host 2001:db8:0:0:0:0:0:1 is not authorized");
assertThatThrownBy(() -> serviceAuthorizationManager.authorize(drwho,
TestProtocol.class, conf, InetAddress.getByName("10.222.0.0")))
.isInstanceOf(AuthorizationException.class);
// an IPv6 client outside of the blocked range is not blocked
assertThatCode(() -> serviceAuthorizationManager.authorize(drwho,
TestProtocol.class, conf, InetAddress.getByName("2001:db9::1")))
.doesNotThrowAnyException();
}

@Test
public void testDefaultBlockedMachineList() throws UnknownHostException {
UserGroupInformation drwho =
Expand Down
153 changes: 153 additions & 0 deletions ...op-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,15 @@
*/
package org.apache.hadoop.util;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collection;
Expand All @@ -42,6 +45,9 @@ public class TestMachineList {
private static String INVALID_CIDR = "10.241/24";
private static String IP_CIDR_LIST =
"10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24";
private static final String IPV6_CIDR_LIST = "2001:db8::/32,fd00:1:2:3::/64";
private static final String IPV4_IPV6_LIST =
"10.222.0.0/16,2001:db8::/32,10.119.103.110,fd00::5";
private static String HOST_LIST = "host1,host4";
private static String HOSTNAME_IP_CIDR_LIST =
"host1,10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24,host4,";
Expand Down Expand Up @@ -310,4 +316,151 @@ public void testGetCollection() {
assertTrue(col.contains(item));
}
}

@Test
public void testIPv4CIDREdgeCases() {
MachineList ml = new MachineList("10.1.2.3/32,10.1.3.2/31", new TestAddressFactory());

assertThat(ml.includes("10.1.2.2")).isFalse();
assertThat(ml.includes("10.1.2.3")).isTrue();
assertThat(ml.includes("10.1.2.4")).isFalse();

assertThat(ml.includes("10.1.3.1")).isFalse();
assertThat(ml.includes("10.1.3.2")).isTrue();
assertThat(ml.includes("10.1.3.3")).isTrue();
assertThat(ml.includes("10.1.3.4")).isFalse();

ml = new MachineList("0.0.0.0/0", new TestAddressFactory());
assertThat(ml.includes("1.2.3.4")).isTrue();
assertThat(ml.includes("255.255.255.255")).isTrue();
assertThat(ml.includes("2001:db8::1")).isFalse();

assertThatThrownBy(() -> new MachineList("10.0.0.0/33", new TestAddressFactory()))
.isInstanceOf(IllegalArgumentException.class);
}

@Test
public void testIPv6List() {
MachineList ml = new MachineList("2001:db8::1,fd00::5", new TestAddressFactory());

assertThat(ml.includes("2001:db8::1")).isTrue();
assertThat(ml.includes("2001:db8:0:0:0:0:0:1")).isTrue();
assertThat(ml.includes("fd00::5")).isTrue();
assertThat(ml.includes("2001:db8::2")).isFalse();
assertThat(ml.includes("10.119.103.112")).isFalse();
}

@Test
public void testIPv6CIDRs() {
MachineList ml = new MachineList(IPV6_CIDR_LIST, new TestAddressFactory());

assertThat(ml.includes("2001:db7:ffff:ffff:ffff:ffff:ffff:ffff")).isFalse();
assertThat(ml.includes("2001:db8::")).isTrue();
assertThat(ml.includes("2001:db8::1")).isTrue();
assertThat(ml.includes("2001:db8:0:0:0:0:0:1")).isTrue();
assertThat(ml.includes("2001:db8:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue();
assertThat(ml.includes("2001:db9::")).isFalse();

assertThat(ml.includes("fd00:1:2:2:ffff:ffff:ffff:ffff")).isFalse();
assertThat(ml.includes("fd00:1:2:3::")).isTrue();
assertThat(ml.includes("fd00:1:2:3:ffff:ffff:ffff:ffff")).isTrue();
assertThat(ml.includes("fd00:1:2:4::")).isFalse();

// host bits in the entry are ignored, as for an IPv4 CIDR
ml = new MachineList("2001:db8::1/32", new TestAddressFactory());
assertThat(ml.includes("2001:db8:ffff::2")).isTrue();
assertThat(ml.includes("2001:db9::1")).isFalse();
}

@Test
public void testIPv6CIDRPrefixNotOnByteBoundary() {
MachineList ml =
new MachineList("2001:db8:8000::/33,fd00:0:0:8::/61", new TestAddressFactory());

assertThat(ml.includes("2001:db8:7fff:ffff:ffff:ffff:ffff:ffff")).isFalse();
assertThat(ml.includes("2001:db8:8000::")).isTrue();
assertThat(ml.includes("2001:db8:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue();
assertThat(ml.includes("2001:db9::")).isFalse();

assertThat(ml.includes("fd00:0:0:7:ffff:ffff:ffff:ffff")).isFalse();
assertThat(ml.includes("fd00:0:0:8::")).isTrue();
assertThat(ml.includes("fd00:0:0:f:ffff:ffff:ffff:ffff")).isTrue();
assertThat(ml.includes("fd00:0:0:10::")).isFalse();
}

@Test
public void testIPv6CIDRWithShortestAndLongestPrefix() {
MachineList ml = new MachineList("2001:db8::1/128", new TestAddressFactory());
assertThat(ml.includes("2001:db8::")).isFalse();
assertThat(ml.includes("2001:db8::1")).isTrue();
assertThat(ml.includes("2001:db8::2")).isFalse();
assertThat(ml.includes("2001:db8::8001")).isFalse();

ml = new MachineList("::/0", new TestAddressFactory());
assertThat(ml.includes("::")).isTrue();
assertThat(ml.includes("2001:db8::1")).isTrue();
assertThat(ml.includes("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue();
assertThat(ml.includes("10.119.103.112")).isFalse();
}

@Test
public void testIPv4AndIPv6CIDRs() {
MachineList ml = new MachineList(IPV4_IPV6_LIST, new TestAddressFactory());

assertThat(ml.includes("10.222.255.255")).isTrue();
assertThat(ml.includes("10.223.0.0")).isFalse();
assertThat(ml.includes("10.119.103.110")).isTrue();
assertThat(ml.includes("2001:db8:ffff::1")).isTrue();
assertThat(ml.includes("2001:db9::1")).isFalse();
assertThat(ml.includes("fd00::5")).isTrue();
assertThat(ml.includes("fd00::6")).isFalse();
}

@Test
public void testCIDRsOfOtherAddressFamily() throws UnknownHostException {
// an IPv6 address is not in an IPv4 range, the lookup must not throw
MachineList ml = new MachineList(CIDR_LIST, new TestAddressFactory());
assertThat(ml.includes("2001:db8::1")).isFalse();
assertThat(ml.includes(InetAddress.getByName("2001:db8::1"))).isFalse();
assertThat(ml.includes("::10.222.0.1")).isFalse();

ml = new MachineList(IPV6_CIDR_LIST, new TestAddressFactory());
assertThat(ml.includes("10.222.0.1")).isFalse();
// 32.1.13.184 has the same leading bytes as 2001:db8::
assertThat(ml.includes(InetAddress.getByName("32.1.13.184"))).isFalse();
}

@Test
public void testIPv4MappedIPv6Address() {
// the JDK turns an IPv4-mapped IPv6 address into an IPv4 address
MachineList ml = new MachineList(CIDR_LIST, new TestAddressFactory());
assertThat(ml.includes("::ffff:10.222.0.1")).isTrue();
assertThat(ml.includes("::ffff:10.223.0.0")).isFalse();

ml = new MachineList("::/0", new TestAddressFactory());
assertThat(ml.includes("::ffff:10.222.0.1")).isFalse();
}

@Test
public void testIPv6CIDRIgnoresScopeIdOfAddress() throws UnknownHostException {
MachineList ml = new MachineList("fe80::/10", new TestAddressFactory());
byte[] linkLocal = InetAddress.getByName("fe80::1").getAddress();
byte[] siteLocal = InetAddress.getByName("fec0::1").getAddress();

assertThat(ml.includes(Inet6Address.getByAddress(null, linkLocal, 1))).isTrue();
assertThat(ml.includes(Inet6Address.getByAddress(null, siteLocal, 1))).isFalse();
}

@Test
public void testInvalidIPv6CIDR() {
String[] invalidCidrs = {"2001:db8::/129", "2001:db8::/1000", "2001:db8::/-1",
"2001:db8::/+32", "2001:db8::/x", "2001:db8::/", "2001:db8::/32/64", "2001:db8::g/32",
":/32", "[2001:db8::]/32", "fe80::%1/64", "fe80::%/64", "2001:db8::\uff11/128",
"::ffff:10.222.0.0/112"};
for (String invalidCidr : invalidCidrs) {
assertThatThrownBy(() -> new MachineList(invalidCidr, new TestAddressFactory()))
.as(invalidCidr)
.isInstanceOf(IllegalArgumentException.class);
}
}
}
Loading