Skip to content

Commit 85be74d

Browse files
committed
refactor(network_namespace): replace async netlink crate with raw netlink sync implementation
1 parent 287c59f commit 85be74d

9 files changed

Lines changed: 829 additions & 245 deletions

File tree

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,5 @@ tmp
2525
ARCHITECTURE.md
2626
scripts/mac.sh
2727
alpine_fs/
28+
scripts/dbg
29+
vim_shorts

Cargo.lock

Lines changed: 18 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ authors = ["pan-ic <dev@pan.fr>"]
1010
anyhow = "1.0.103"
1111
clap = { version = "4.6.1", features = ["derive"] }
1212
futures = "0.3"
13-
libc = "0.2.186"
14-
nix = { version = "0.31.3", features = ["fs", "hostname", "mount", "process", "sched", "signal"] }
13+
libc = "0.2.189"
14+
nix = { version = "0.31.3", features = ["fs", "hostname", "mount", "process", "sched", "signal", "socket"] }
1515
rtnetlink = "0.21.0"
1616
thiserror = "2.0.18"
1717
tokio = { version = "1.53.1", features = ["rt", "rt-multi-thread"] }

DEVLOG.md

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,3 +64,31 @@ Apparently netlink_packet_route::link::LinkMessage exist behind packet::route::L
6464
Implementation using rtnetlink blocks on child setup because of AsyncSockets, in theory it could be implemented to finish the experiment but I use
6565
raw socket as final implementation so I've mixed rtnetlink for parent + nsenter for child. Child would use exactly the same init step than parent but
6666
after calling new_connection_with_socket() that is the remote connection to the child network.
67+
68+
## 2026/07/25-31 Hard session
69+
Last part of the network core was to directly use raw socket and Kernel structs. I would qualify that first experience with Kernel network programming
70+
as both passionate and frustrating. Frustration comes from:
71+
- Often real difficulties to find sources about how things works. Documentation is often really poor if you don't know where to find it. So the rule I've
72+
established for Kernel network programming (will be extended to Kernel programming) sourcing:
73+
a. man pages; if lucky, get the big picture of the data struct and implementation with the related pages
74+
(e.g. netlink(7)-(3), rtnetlink(7)-(3)...),
75+
b. github, OSS projects; if lucky, any public source code of a related project used to check syscalls and implementation
76+
(e.g. runc, containerd, rtnetlink rust crate, netlink go package, iproute2..),
77+
c. strace/perf/bpftrace
78+
(e.g. strace -x -s 1000 -e sendmsg ip addr add 10.0.0.1/24 dev veth1 2>&1),
79+
d. minimal program; just write a minimal program that aims to use the tageted datastruct, that allows to focus/divide and give a better exploitation
80+
of the potential responses of the API/Kernel
81+
e. Kernel source code (bootlin), grep
82+
(e.g. bootlin '/net' 'drivers/net' '/include', 'include/uapi'; grep -r {} "/usr/include/linux"),
83+
f. Kernel documentation at /Documentation, examples at /samples,
84+
g. LWN.net for articles on Kernel features.
85+
The faster method to me is to use strace then check docs/code.
86+
- A lot of type casting in this kind of code, because of the use of Rust types and C types and the switch between stdlib, libc and netlink
87+
type casting hell
88+
- Sometimes C macros are not yet translated in Rust lib, grep -r "{}" /usr/include/linux is a life saver to get the scalar value there.
89+
I've also been tricked by two things:
90+
- socket messages queue, unread ACK made a mess with get_if_id() giving a stale id.
91+
- when you use an C struct implemented in Rust and that implementation has private fields these have to be initialized (zeroed padding) using methods like
92+
std::mem::zeroed()
93+
At the end implementing from scratch had the great advantage to make the child network setup easier, it only needs to be accessed by opening an fd on
94+
the child net ns then uses of setns to switch from parent to child, setting up and vice versa.

README.md

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,10 @@ https://github.com/opencontainers/runc
3131

3232
https://github.com/youki-dev/youki
3333

34-
https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/cgroups.html
34+
https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/cgroups.html
35+
36+
https://books.google.fr/books/about/Linux_Kernel_Networking.html?id=96V4AgAAQBAJ&redir_esc=y
37+
38+
https://github.com/iproute2/iproute2
39+
40+
https://elixir.bootlin.com/linux/v6.19.14/source

scripts/strace_format

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/usr/bin/python3
2+
import sys
3+
4+
for line in sys.stdin:
5+
if 'sendmsg' in line:
6+
print(line.replace(', ', ',\n ').replace('[[', '[\n ['))

src/container.rs

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
use anyhow::Context;
2+
#[cfg(target_os = "linux")]
3+
use nix::sched::{CloneCb, CloneFlags, clone, setns};
4+
use nix::{
5+
mount::{MntFlags, MsFlags, mount, umount2},
6+
sys::wait::waitpid,
7+
unistd::{Pid, chdir, execve, getpid, pivot_root, sethostname, write},
8+
};
9+
use std::ffi::{CString, c_int};
10+
use std::fs::{
11+
create_dir_all,
12+
File,
13+
remove_dir,
14+
};
15+
use std::path::Path;
16+
use std::os::fd::AsFd;
17+
use std::net::Ipv4Addr;
18+
19+
use crate::network::{
20+
add_default_route,
21+
get_interface_index,
22+
create_netlink_socket,
23+
create_veth_pair,
24+
move_to_netns,
25+
set_interface_up,
26+
set_ip_addr,
27+
Writer,
28+
};
29+
30+
macro_rules! child_try {
31+
($expr:expr, $msg:expr, $eval:expr) => {
32+
if let Err(e) = $expr {
33+
let msg = format!("{}: {}\n", $msg, e);
34+
let _ = write(std::io::stderr(), msg.as_bytes());
35+
unsafe {
36+
libc::_exit($eval as c_int);
37+
}
38+
}
39+
};
40+
}
41+
42+
#[allow(unreachable_code)]
43+
#[cfg(target_os = "linux")]
44+
pub fn create_child_process(name: &str, cmd: &str) -> anyhow::Result<()> {
45+
//execve() call variables
46+
let path = CString::new("/bin/sh").context("c_str failure")?;
47+
let ca = [
48+
CString::new("sh").context("c_str failure")?,
49+
CString::new("-c").context("c_str failure")?,
50+
CString::new(cmd).context("c_str failure")?,
51+
];
52+
let parent_pid = getpid();
53+
let child_pid: Pid;
54+
55+
//mount() + pivot_root() call variables
56+
//TODO: replace what can be repaced by config parsing
57+
//note that variable might move in the clone call back if not needed elsewhere (and so, won't have
58+
//to be static), will be determined during refacto
59+
let new_root: &'static str = "/home/debian/clonebox/alpine_fs";
60+
let mount_proc: &'static str = "proc";
61+
let mount_proc_path: &'static str = "/proc";
62+
let put_old: &'static str = "/put_old";
63+
let child_old_path: &'static str = "/home/debian/clonebox/alpine_fs/put_old";
64+
65+
//clone() call variables
66+
let cb: CloneCb = Box::new(|| -> isize {
67+
//child mount has to be private else it's still shared with the parent
68+
child_try!(
69+
mount(
70+
None::<&str>,
71+
"/",
72+
None::<&str>,
73+
MsFlags::MS_PRIVATE | MsFlags::MS_REC,
74+
None::<&str>
75+
),
76+
"private mount",
77+
1
78+
);
79+
80+
//change hostname
81+
child_try!(sethostname(name), "sethostname", 1);
82+
83+
//bind mount fs
84+
child_try!(
85+
mount(
86+
Some(new_root),
87+
new_root,
88+
None::<&str>,
89+
MsFlags::MS_BIND,
90+
None::<&str>
91+
),
92+
"bind mount",
93+
1
94+
);
95+
96+
//pivot_root()
97+
child_try!(
98+
create_dir_all(Path::new(&child_old_path)),
99+
"create_dir_all",
100+
1
101+
);
102+
child_try!(pivot_root(new_root, child_old_path), "pivot_root", 1);
103+
child_try!(chdir("/"), "chdir", 1);
104+
child_try!(umount2(put_old, MntFlags::MNT_DETACH), "unmount2", 1);
105+
child_try!(remove_dir(Path::new(put_old)), "remove_dir", 1);
106+
107+
//mount proc
108+
child_try!(
109+
mount(
110+
Some(mount_proc),
111+
mount_proc_path,
112+
Some(mount_proc),
113+
MsFlags::empty(),
114+
None::<&str>
115+
),
116+
"fs mount",
117+
1
118+
);
119+
120+
let Err(e) = execve(&path, &ca, &[] as &[CString]);
121+
let e = format!("execve failed: {}\n", e);
122+
//write might fail but we are about to exit anyway
123+
let _ = write(std::io::stderr(), e.as_bytes());
124+
unsafe {
125+
libc::_exit(1);
126+
}
127+
0
128+
});
129+
130+
//clone() call variables
131+
let mut stack = vec![0u8; 1024 * 1024];
132+
let clone_flags: CloneFlags = CloneFlags::CLONE_NEWPID
133+
| CloneFlags::CLONE_NEWUTS
134+
| CloneFlags::CLONE_NEWNS
135+
| CloneFlags::CLONE_NEWNET;
136+
let signal: Option<c_int> = Some(libc::SIGCHLD);
137+
138+
println!("Parent pid is {}", parent_pid);
139+
140+
unsafe {
141+
child_pid = clone(cb, &mut stack, clone_flags, signal).context("clone failure")?;
142+
}
143+
144+
let host_ns_fd = File::open("/proc/self/ns/net")?;
145+
let peer_ns_fd = File::open(format!("/proc/{}/ns/net", child_pid.as_raw()))?;
146+
let host = "veth1";
147+
let host_address = Ipv4Addr::new(10, 0, 0, 1);
148+
let peer_address = Ipv4Addr::new(10, 0, 0, 2);
149+
let peer = "veth1_peer";
150+
let mut w = Writer {
151+
buf : Vec::new(),
152+
};
153+
let host_sk = create_netlink_socket()?;
154+
let _ = create_veth_pair(host_sk.as_fd(), &mut w, host, peer)?;
155+
let host_i_id = get_interface_index(host_sk.as_fd(), &mut w, host)?;
156+
let _ = set_ip_addr(host_sk.as_fd(), &mut w, host_i_id, host_address, 24u8)?;
157+
let _ = set_interface_up(host_sk.as_fd(), &mut w, host_i_id)?;
158+
let child_i_id = get_interface_index(host_sk.as_fd(), &mut w, peer)?;
159+
let _ = move_to_netns(host_sk.as_fd(), &mut w, &child_i_id, &peer_ns_fd)?;
160+
161+
let _ = setns(peer_ns_fd.as_fd(), CloneFlags::CLONE_NEWNET)?;
162+
163+
let child_sk = create_netlink_socket()?;
164+
let _ = set_ip_addr(child_sk.as_fd(), &mut w, child_i_id, peer_address, 24u8)?;
165+
let _ = set_interface_up(child_sk.as_fd(), &mut w, child_i_id)?;
166+
let _ = set_interface_up(child_sk.as_fd(), &mut w, 1)?;
167+
let _ = add_default_route(child_sk.as_fd(), &mut w, host_address)?;
168+
169+
drop(child_sk);
170+
171+
let _ = setns(host_ns_fd.as_fd(), CloneFlags::CLONE_NEWNET)?;
172+
173+
println!("Child pid is {}", child_pid);
174+
let child_return = waitpid(child_pid, None).context("waitpid failure")?;
175+
println!("Child return is: {:?}", child_return);
176+
177+
Ok(())
178+
}

0 commit comments

Comments
 (0)