From c444e5d75dc87cf7e486c0e4273413e83e385d5e Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Wed, 17 Jun 2026 15:35:35 +0800 Subject: [PATCH 01/21] Integrate Firecracker UFFD mapping receive --- memory/manager/snapshot_state.go | 21 +-- memory/manager/uffd_mapping.go | 78 +++++++++ memory/manager/uffd_mapping_test.go | 144 +++++++++++++++++ memory/manager/uffd_socket.go | 94 +++++++++++ memory/manager/uffd_socket_test.go | 238 ++++++++++++++++++++++++++++ 5 files changed, 565 insertions(+), 10 deletions(-) create mode 100644 memory/manager/uffd_mapping.go create mode 100644 memory/manager/uffd_mapping_test.go create mode 100644 memory/manager/uffd_socket.go create mode 100644 memory/manager/uffd_socket_test.go diff --git a/memory/manager/snapshot_state.go b/memory/manager/snapshot_state.go index 0aecaaf16..de8462a55 100644 --- a/memory/manager/snapshot_state.go +++ b/memory/manager/snapshot_state.go @@ -40,7 +40,6 @@ import ( "syscall" "time" - "github.com/ftrvxmtrx/fd" log "github.com/sirupsen/logrus" "golang.org/x/sys/unix" @@ -67,12 +66,13 @@ type SnapshotStateCfg struct { // of the VM. type SnapshotState struct { SnapshotStateCfg - firstPageFaultOnce *sync.Once // to initialize the start virtual address and replay - startAddress uint64 - userFaultFD *os.File - trace *Trace - epfd int - quitCh chan int + firstPageFaultOnce *sync.Once // to initialize the start virtual address and replay + startAddress uint64 + userFaultFD *os.File + guestRegionMappings []GuestRegionUffdMapping + trace *Trace + epfd int + quitCh chan int // to indicate whether the instance has even been activated. this is to // get around cases where offload is called for the first time @@ -145,13 +145,14 @@ func (s *SnapshotState) getUFFD() error { sendfdConn := c.(*net.UnixConn) - fs, err := fd.Get(sendfdConn, 1, []string{"a file"}) + mappings, userFaultFD, err := receiveUffdMappingsAndFD(sendfdConn) if err != nil { - log.Error("Failed to receive the uffd") + log.Error("Failed to receive the uffd and guest memory mappings") return err } - s.userFaultFD = fs[0] + s.guestRegionMappings = mappings + s.userFaultFD = userFaultFD return nil } diff --git a/memory/manager/uffd_mapping.go b/memory/manager/uffd_mapping.go new file mode 100644 index 000000000..b5b23b148 --- /dev/null +++ b/memory/manager/uffd_mapping.go @@ -0,0 +1,78 @@ +package manager + +import ( + "errors" + "fmt" + "math" +) + +var ( + errInvalidGuestRegionPageSize = errors.New("guest region page size must be non-zero") + errGuestRegionNotFound = errors.New("fault address is outside guest memory mappings") +) + +// GuestRegionUffdMapping describes Firecracker's UFFD guest memory mapping. +type GuestRegionUffdMapping struct { + BaseHostVirtAddr uint64 `json:"base_host_virt_addr"` + Size uint64 `json:"size"` + Offset uint64 `json:"offset"` + PageSize uint64 `json:"page_size"` +} + +func pageAlignFaultAddress(faultAddr uint64, region GuestRegionUffdMapping) (uint64, error) { + if region.PageSize == 0 { + return 0, errInvalidGuestRegionPageSize + } + + return faultAddr - faultAddr%region.PageSize, nil +} + +func findGuestRegionForFaultPage(regions []GuestRegionUffdMapping, faultPageAddr uint64) (GuestRegionUffdMapping, error) { + for _, region := range regions { + if regionContainsFaultPage(region, faultPageAddr) { + return region, nil + } + } + + return GuestRegionUffdMapping{}, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultPageAddr) +} + +func guestMemoryOffsetForFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) (uint64, error) { + if region.PageSize == 0 { + return 0, errInvalidGuestRegionPageSize + } + if !regionContainsFaultPage(region, faultPageAddr) { + return 0, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultPageAddr) + } + + regionOffset := faultPageAddr - region.BaseHostVirtAddr + if region.Offset > math.MaxUint64-regionOffset { + return 0, fmt.Errorf("guest memory offset overflow for fault address %#x", faultPageAddr) + } + + return region.Offset + regionOffset, nil +} + +func guestMemoryOffsetForFault(regions []GuestRegionUffdMapping, faultAddr uint64) (uint64, error) { + for _, region := range regions { + faultPageAddr, err := pageAlignFaultAddress(faultAddr, region) + if err != nil { + return 0, err + } + if !regionContainsFaultPage(region, faultPageAddr) { + continue + } + + return guestMemoryOffsetForFaultPage(region, faultPageAddr) + } + + return 0, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultAddr) +} + +func regionContainsFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) bool { + if region.Size == 0 || faultPageAddr < region.BaseHostVirtAddr { + return false + } + + return faultPageAddr-region.BaseHostVirtAddr < region.Size +} diff --git a/memory/manager/uffd_mapping_test.go b/memory/manager/uffd_mapping_test.go new file mode 100644 index 000000000..605ba9921 --- /dev/null +++ b/memory/manager/uffd_mapping_test.go @@ -0,0 +1,144 @@ +package manager + +import ( + "errors" + "testing" +) + +func TestPageAlignFaultAddress(t *testing.T) { + region := GuestRegionUffdMapping{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + PageSize: 0x1000, + } + + got, err := pageAlignFaultAddress(0x101234, region) + if err != nil { + t.Fatalf("pageAlignFaultAddress returned error: %v", err) + } + if want := uint64(0x101000); got != want { + t.Fatalf("pageAlignFaultAddress() = %#x, want %#x", got, want) + } +} + +func TestFindGuestRegionForFaultPage(t *testing.T) { + regions := []GuestRegionUffdMapping{ + { + BaseHostVirtAddr: 0x100000, + Size: 0x2000, + PageSize: 0x1000, + }, + { + BaseHostVirtAddr: 0x200000, + Size: 0x3000, + Offset: 0x8000, + PageSize: 0x1000, + }, + } + + got, err := findGuestRegionForFaultPage(regions, 0x201000) + if err != nil { + t.Fatalf("findGuestRegionForFaultPage returned error: %v", err) + } + if want := regions[1]; got != want { + t.Fatalf("findGuestRegionForFaultPage() = %+v, want %+v", got, want) + } +} + +func TestGuestMemoryOffsetForFault(t *testing.T) { + tests := []struct { + name string + regions []GuestRegionUffdMapping + fault uint64 + want uint64 + }{ + { + name: "one region with zero offset", + regions: []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + PageSize: 0x1000, + }}, + fault: 0x102000, + want: 0x2000, + }, + { + name: "one region with non-zero offset", + regions: []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + Offset: 0x800000, + PageSize: 0x1000, + }}, + fault: 0x103000, + want: 0x803000, + }, + { + name: "multiple regions", + regions: []GuestRegionUffdMapping{ + { + BaseHostVirtAddr: 0x100000, + Size: 0x2000, + PageSize: 0x1000, + }, + { + BaseHostVirtAddr: 0x200000, + Size: 0x3000, + Offset: 0x900000, + PageSize: 0x1000, + }, + }, + fault: 0x201000, + want: 0x901000, + }, + { + name: "address not page-aligned", + regions: []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + Offset: 0x300000, + PageSize: 0x1000, + }}, + fault: 0x101234, + want: 0x301000, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := guestMemoryOffsetForFault(tt.regions, tt.fault) + if err != nil { + t.Fatalf("guestMemoryOffsetForFault returned error: %v", err) + } + if got != tt.want { + t.Fatalf("guestMemoryOffsetForFault() = %#x, want %#x", got, tt.want) + } + }) + } +} + +func TestGuestMemoryOffsetForFaultOutsideAllRegions(t *testing.T) { + regions := []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x2000, + PageSize: 0x1000, + }} + + _, err := guestMemoryOffsetForFault(regions, 0x103000) + if !errors.Is(err, errGuestRegionNotFound) { + t.Fatalf("guestMemoryOffsetForFault() error = %v, want %v", err, errGuestRegionNotFound) + } +} + +func TestGuestMemoryOffsetForFaultZeroPageSize(t *testing.T) { + regions := []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x2000, + PageSize: 0, + }} + + _, err := guestMemoryOffsetForFault(regions, 0x100000) + if !errors.Is(err, errInvalidGuestRegionPageSize) { + t.Fatalf("guestMemoryOffsetForFault() error = %v, want %v", err, errInvalidGuestRegionPageSize) + } +} diff --git a/memory/manager/uffd_socket.go b/memory/manager/uffd_socket.go new file mode 100644 index 000000000..03bcfc832 --- /dev/null +++ b/memory/manager/uffd_socket.go @@ -0,0 +1,94 @@ +package manager + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "os" + "time" + + "golang.org/x/sys/unix" +) + +const ( + uffdSocketPayloadSize = 64 * 1024 + uffdSocketFDLimit = 2 + uffdSocketReadTimeout = time.Second +) + +var ( + errUnexpectedUffdFDCount = errors.New("expected exactly one uffd fd") + errNoGuestRegionMappings = errors.New("no guest region mappings received") +) + +func receiveUffdMappingsAndFD(conn *net.UnixConn) ([]GuestRegionUffdMapping, *os.File, error) { + if err := conn.SetReadDeadline(time.Now().Add(uffdSocketReadTimeout)); err != nil { + return nil, nil, err + } + defer func() { _ = conn.SetReadDeadline(time.Time{}) }() + + body := make([]byte, uffdSocketPayloadSize) + oob := make([]byte, unix.CmsgSpace(uffdSocketFDLimit*4)) + + n, oobn, flags, _, err := conn.ReadMsgUnix(body, oob) + if err != nil { + return nil, nil, err + } + if flags&unix.MSG_TRUNC != 0 { + return nil, nil, errors.New("uffd mappings payload was truncated") + } + if flags&unix.MSG_CTRUNC != 0 { + return nil, nil, errors.New("uffd fd control message was truncated") + } + + fds, err := parseUnixRights(oob[:oobn]) + if err != nil { + return nil, nil, err + } + if len(fds) != 1 { + closeFDs(fds) + return nil, nil, fmt.Errorf("%w: got %d", errUnexpectedUffdFDCount, len(fds)) + } + + uffdFile := os.NewFile(uintptr(fds[0]), "userfaultfd") + if uffdFile == nil { + return nil, nil, errors.New("failed to create file for uffd fd") + } + + var mappings []GuestRegionUffdMapping + if err := json.Unmarshal(body[:n], &mappings); err != nil { + _ = uffdFile.Close() + return nil, nil, fmt.Errorf("cannot deserialize memory mappings: %w", err) + } + if len(mappings) == 0 { + _ = uffdFile.Close() + return nil, nil, errNoGuestRegionMappings + } + + return mappings, uffdFile, nil +} + +func parseUnixRights(oob []byte) ([]int, error) { + scms, err := unix.ParseSocketControlMessage(oob) + if err != nil { + return nil, err + } + + var fds []int + for i := range scms { + rights, err := unix.ParseUnixRights(&scms[i]) + if err != nil { + return nil, err + } + fds = append(fds, rights...) + } + + return fds, nil +} + +func closeFDs(fds []int) { + for _, receivedFD := range fds { + _ = unix.Close(receivedFD) + } +} diff --git a/memory/manager/uffd_socket_test.go b/memory/manager/uffd_socket_test.go new file mode 100644 index 000000000..7483443fc --- /dev/null +++ b/memory/manager/uffd_socket_test.go @@ -0,0 +1,238 @@ +package manager + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "reflect" + "testing" + + "golang.org/x/sys/unix" +) + +func TestReceiveUffdMappingsAndFD(t *testing.T) { + mappings := []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + Offset: 0x8000, + PageSize: 0x1000, + }} + body, err := json.Marshal(mappings) + if err != nil { + t.Fatalf("json.Marshal returned error: %v", err) + } + + sentFile := tempFileWithContent(t, "uffd fd payload") + gotMappings, gotFile, err := receiveFromTestSocket(t, func(conn *net.UnixConn) error { + return writeUffdSocketPayload(conn, body, sentFile) + }) + if err != nil { + t.Fatalf("receiveUffdMappingsAndFD returned error: %v", err) + } + defer func() { _ = gotFile.Close() }() + + if !reflect.DeepEqual(gotMappings, mappings) { + t.Fatalf("receiveUffdMappingsAndFD mappings = %+v, want %+v", gotMappings, mappings) + } + + gotPayload, err := io.ReadAll(gotFile) + if err != nil { + t.Fatalf("reading received fd returned error: %v", err) + } + if want := "uffd fd payload"; string(gotPayload) != want { + t.Fatalf("received fd payload = %q, want %q", gotPayload, want) + } +} + +func TestReceiveUffdMappingsAndFDInvalidJSON(t *testing.T) { + sentFile := tempFileWithContent(t, "unused") + _, gotFile, err := receiveFromTestSocket(t, func(conn *net.UnixConn) error { + return writeUffdSocketPayload(conn, []byte("{"), sentFile) + }) + if gotFile != nil { + _ = gotFile.Close() + } + if err == nil { + t.Fatal("receiveUffdMappingsAndFD succeeded for invalid JSON") + } +} + +func TestReceiveUffdMappingsAndFDMissingFD(t *testing.T) { + body := validMappingsJSON(t) + _, gotFile, err := receiveFromTestSocket(t, func(conn *net.UnixConn) error { + return writeUffdSocketPayload(conn, body) + }) + if gotFile != nil { + _ = gotFile.Close() + } + if !errors.Is(err, errUnexpectedUffdFDCount) { + t.Fatalf("receiveUffdMappingsAndFD error = %v, want %v", err, errUnexpectedUffdFDCount) + } +} + +func TestReceiveUffdMappingsAndFDRejectsMultipleFDs(t *testing.T) { + body := validMappingsJSON(t) + firstFile := tempFileWithContent(t, "first") + secondFile := tempFileWithContent(t, "second") + + _, gotFile, err := receiveFromTestSocket(t, func(conn *net.UnixConn) error { + return writeUffdSocketPayload(conn, body, firstFile, secondFile) + }) + if gotFile != nil { + _ = gotFile.Close() + } + if !errors.Is(err, errUnexpectedUffdFDCount) { + t.Fatalf("receiveUffdMappingsAndFD error = %v, want %v", err, errUnexpectedUffdFDCount) + } +} + +func TestSnapshotStateGetUFFDStoresMappingsAndFD(t *testing.T) { + mappings := []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x400000, + Size: 0x2000, + PageSize: 0x1000, + }} + body, err := json.Marshal(mappings) + if err != nil { + t.Fatalf("json.Marshal returned error: %v", err) + } + sentFile := tempFileWithContent(t, "state fd payload") + + socketPath := filepath.Join(t.TempDir(), "uffd.sock") + listener, err := net.ListenUnix("unix", &net.UnixAddr{Name: socketPath, Net: "unix"}) + if err != nil { + t.Fatalf("net.ListenUnix returned error: %v", err) + } + defer func() { _ = listener.Close() }() + + serverErrCh := make(chan error, 1) + go func() { + conn, err := listener.AcceptUnix() + if err != nil { + serverErrCh <- err + return + } + defer func() { _ = conn.Close() }() + serverErrCh <- writeUffdSocketPayload(conn, body, sentFile) + }() + + state := &SnapshotState{ + SnapshotStateCfg: SnapshotStateCfg{ + InstanceSockAddr: socketPath, + }, + } + if err := state.getUFFD(); err != nil { + t.Fatalf("getUFFD returned error: %v", err) + } + defer func() { _ = state.userFaultFD.Close() }() + + if err := <-serverErrCh; err != nil { + t.Fatalf("test server returned error: %v", err) + } + if !reflect.DeepEqual(state.guestRegionMappings, mappings) { + t.Fatalf("guestRegionMappings = %+v, want %+v", state.guestRegionMappings, mappings) + } + gotPayload, err := io.ReadAll(state.userFaultFD) + if err != nil { + t.Fatalf("reading state.userFaultFD returned error: %v", err) + } + if want := "state fd payload"; string(gotPayload) != want { + t.Fatalf("state.userFaultFD payload = %q, want %q", gotPayload, want) + } +} + +func receiveFromTestSocket(t *testing.T, send func(*net.UnixConn) error) ([]GuestRegionUffdMapping, *os.File, error) { + t.Helper() + + socketPath := filepath.Join(t.TempDir(), "uffd.sock") + addr := &net.UnixAddr{Name: socketPath, Net: "unix"} + listener, err := net.ListenUnix("unix", addr) + if err != nil { + t.Fatalf("net.ListenUnix returned error: %v", err) + } + defer func() { _ = listener.Close() }() + + sendErrCh := make(chan error, 1) + go func() { + conn, err := net.DialUnix("unix", nil, addr) + if err != nil { + sendErrCh <- err + return + } + defer func() { _ = conn.Close() }() + sendErrCh <- send(conn) + }() + + conn, err := listener.AcceptUnix() + if err != nil { + t.Fatalf("AcceptUnix returned error: %v", err) + } + defer func() { _ = conn.Close() }() + + mappings, file, receiveErr := receiveUffdMappingsAndFD(conn) + if sendErr := <-sendErrCh; sendErr != nil { + t.Fatalf("test sender returned error: %v", sendErr) + } + + return mappings, file, receiveErr +} + +func writeUffdSocketPayload(conn *net.UnixConn, body []byte, files ...*os.File) error { + fds := make([]int, 0, len(files)) + for _, file := range files { + fds = append(fds, int(file.Fd())) + } + + var oob []byte + if len(fds) > 0 { + oob = unix.UnixRights(fds...) + } + + n, _, err := conn.WriteMsgUnix(body, oob, nil) + if err != nil { + return err + } + if n != len(body) { + return fmt.Errorf("sent %d bytes, want %d", n, len(body)) + } + + return nil +} + +func validMappingsJSON(t *testing.T) []byte { + t.Helper() + + body, err := json.Marshal([]GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x1000, + PageSize: 0x1000, + }}) + if err != nil { + t.Fatalf("json.Marshal returned error: %v", err) + } + + return body +} + +func tempFileWithContent(t *testing.T, content string) *os.File { + t.Helper() + + file, err := os.CreateTemp(t.TempDir(), "uffd-fd-*") + if err != nil { + t.Fatalf("os.CreateTemp returned error: %v", err) + } + t.Cleanup(func() { _ = file.Close() }) + + if _, err := file.WriteString(content); err != nil { + t.Fatalf("WriteString returned error: %v", err) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + t.Fatalf("Seek returned error: %v", err) + } + + return file +} From 8f9901666db4017f6e33406b0c241d518f56bc36 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Wed, 17 Jun 2026 16:06:41 +0800 Subject: [PATCH 02/21] Use Firecracker mappings for UFFD faults --- memory/manager/snapshot_state.go | 35 ++++++++--- memory/manager/uffd_mapping.go | 33 ++++++++++ memory/manager/uffd_mapping_test.go | 98 +++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 8 deletions(-) diff --git a/memory/manager/snapshot_state.go b/memory/manager/snapshot_state.go index de8462a55..1f6dc3247 100644 --- a/memory/manager/snapshot_state.go +++ b/memory/manager/snapshot_state.go @@ -396,14 +396,18 @@ func (s *SnapshotState) servePageFault(fd int, address uint64) error { return nil } - offset := address - s.startAddress + copyArgs, err := pageFaultCopyArgsForFault(s.guestRegionMappings, address) + if err != nil { + return err + } - src := uint64(uintptr(unsafe.Pointer(&s.guestMem[offset]))) - dst := uint64(int64(address) & ^(int64(os.Getpagesize()) - 1)) - mode := uint64(0) + src, err := guestMemPointer(s.guestMem, copyArgs.srcOffset, copyArgs.copyLen) + if err != nil { + return err + } rec := Record{ - offset: offset, + offset: copyArgs.srcOffset, } if !s.isRecordReady { @@ -428,7 +432,7 @@ func (s *SnapshotState) servePageFault(fd int, address uint64) error { tStart = time.Now() } - err := installRegion(fd, src, dst, mode, 1) + err = installRegionBytes(fd, src, copyArgs.dstAddr, copyArgs.copyMode, copyArgs.copyLen) if s.metricsModeOn { s.currentMetric.MetricMap[serveUniqueMetric] += metrics.ToUS(time.Since(tStart)) @@ -468,13 +472,17 @@ func (s *SnapshotState) installWorkingSetPages(fd int) { wake(fd, s.startAddress, os.Getpagesize()) } -func installRegion(fd int, src, dst, mode, len uint64) error { +func installRegion(fd int, src, dst, mode, pageCount uint64) error { + return installRegionBytes(fd, src, dst, mode, uint64(os.Getpagesize())*pageCount) +} + +func installRegionBytes(fd int, src, dst, mode, length uint64) error { cUC := C.struct_uffdio_copy{ mode: C.ulonglong(mode), copy: 0, src: C.ulonglong(src), dst: C.ulonglong(dst), - len: C.ulonglong(uint64(os.Getpagesize()) * len), + len: C.ulonglong(length), } err := ioctl(uintptr(fd), int(C.const_UFFDIO_COPY), unsafe.Pointer(&cUC)) @@ -485,6 +493,17 @@ func installRegion(fd int, src, dst, mode, len uint64) error { return nil } +func guestMemPointer(guestMem []byte, offset, length uint64) (uint64, error) { + if length == 0 { + return 0, errors.New("guest memory copy length must be non-zero") + } + if offset >= uint64(len(guestMem)) || length > uint64(len(guestMem))-offset { + return 0, fmt.Errorf("guest memory copy is outside mapped file: offset=%#x len=%#x size=%#x", offset, length, len(guestMem)) + } + + return uint64(uintptr(unsafe.Pointer(&guestMem[int(offset)]))), nil +} + func ioctl(fd uintptr, request int, argp unsafe.Pointer) error { _, _, errno := unix.Syscall( unix.SYS_IOCTL, diff --git a/memory/manager/uffd_mapping.go b/memory/manager/uffd_mapping.go index b5b23b148..853d5c90c 100644 --- a/memory/manager/uffd_mapping.go +++ b/memory/manager/uffd_mapping.go @@ -19,6 +19,13 @@ type GuestRegionUffdMapping struct { PageSize uint64 `json:"page_size"` } +type pageFaultCopyArgs struct { + srcOffset uint64 + dstAddr uint64 + copyLen uint64 + copyMode uint64 +} + func pageAlignFaultAddress(faultAddr uint64, region GuestRegionUffdMapping) (uint64, error) { if region.PageSize == 0 { return 0, errInvalidGuestRegionPageSize @@ -69,6 +76,32 @@ func guestMemoryOffsetForFault(regions []GuestRegionUffdMapping, faultAddr uint6 return 0, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultAddr) } +func pageFaultCopyArgsForFault(regions []GuestRegionUffdMapping, faultAddr uint64) (pageFaultCopyArgs, error) { + for _, region := range regions { + faultPageAddr, err := pageAlignFaultAddress(faultAddr, region) + if err != nil { + return pageFaultCopyArgs{}, err + } + if !regionContainsFaultPage(region, faultPageAddr) { + continue + } + + srcOffset, err := guestMemoryOffsetForFaultPage(region, faultPageAddr) + if err != nil { + return pageFaultCopyArgs{}, err + } + + return pageFaultCopyArgs{ + srcOffset: srcOffset, + dstAddr: faultPageAddr, + copyLen: region.PageSize, + copyMode: 0, + }, nil + } + + return pageFaultCopyArgs{}, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultAddr) +} + func regionContainsFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) bool { if region.Size == 0 || faultPageAddr < region.BaseHostVirtAddr { return false diff --git a/memory/manager/uffd_mapping_test.go b/memory/manager/uffd_mapping_test.go index 605ba9921..2b1203a95 100644 --- a/memory/manager/uffd_mapping_test.go +++ b/memory/manager/uffd_mapping_test.go @@ -142,3 +142,101 @@ func TestGuestMemoryOffsetForFaultZeroPageSize(t *testing.T) { t.Fatalf("guestMemoryOffsetForFault() error = %v, want %v", err, errInvalidGuestRegionPageSize) } } + +func TestPageFaultCopyArgsForFault(t *testing.T) { + tests := []struct { + name string + regions []GuestRegionUffdMapping + fault uint64 + want pageFaultCopyArgs + }{ + { + name: "zero offset", + regions: []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + PageSize: 0x1000, + }}, + fault: 0x102000, + want: pageFaultCopyArgs{ + srcOffset: 0x2000, + dstAddr: 0x102000, + copyLen: 0x1000, + copyMode: 0, + }, + }, + { + name: "non-zero offset", + regions: []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + Offset: 0x800000, + PageSize: 0x1000, + }}, + fault: 0x103000, + want: pageFaultCopyArgs{ + srcOffset: 0x803000, + dstAddr: 0x103000, + copyLen: 0x1000, + copyMode: 0, + }, + }, + { + name: "not page-aligned", + regions: []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + Offset: 0x300000, + PageSize: 0x1000, + }}, + fault: 0x101234, + want: pageFaultCopyArgs{ + srcOffset: 0x301000, + dstAddr: 0x101000, + copyLen: 0x1000, + copyMode: 0, + }, + }, + { + name: "larger page size", + regions: []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x8000, + Offset: 0x500000, + PageSize: 0x2000, + }}, + fault: 0x103456, + want: pageFaultCopyArgs{ + srcOffset: 0x502000, + dstAddr: 0x102000, + copyLen: 0x2000, + copyMode: 0, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := pageFaultCopyArgsForFault(tt.regions, tt.fault) + if err != nil { + t.Fatalf("pageFaultCopyArgsForFault returned error: %v", err) + } + if got != tt.want { + t.Fatalf("pageFaultCopyArgsForFault() = %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestPageFaultCopyArgsForFaultOutsideAllRegions(t *testing.T) { + regions := []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x2000, + PageSize: 0x1000, + }} + + _, err := pageFaultCopyArgsForFault(regions, 0x103000) + if !errors.Is(err, errGuestRegionNotFound) { + t.Fatalf("pageFaultCopyArgsForFault() error = %v, want %v", err, errGuestRegionNotFound) + } +} From a5ddc4190ea003b3421ca230255430557d6029f0 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Fri, 19 Jun 2026 14:31:19 +0800 Subject: [PATCH 03/21] Clean up UFFD mapping receive tests --- memory/manager/uffd_mapping.go | 15 ++++----------- memory/manager/uffd_socket.go | 1 + memory/manager/uffd_socket_test.go | 13 +++++++++++++ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/memory/manager/uffd_mapping.go b/memory/manager/uffd_mapping.go index 853d5c90c..0458fb0d9 100644 --- a/memory/manager/uffd_mapping.go +++ b/memory/manager/uffd_mapping.go @@ -61,19 +61,12 @@ func guestMemoryOffsetForFaultPage(region GuestRegionUffdMapping, faultPageAddr } func guestMemoryOffsetForFault(regions []GuestRegionUffdMapping, faultAddr uint64) (uint64, error) { - for _, region := range regions { - faultPageAddr, err := pageAlignFaultAddress(faultAddr, region) - if err != nil { - return 0, err - } - if !regionContainsFaultPage(region, faultPageAddr) { - continue - } - - return guestMemoryOffsetForFaultPage(region, faultPageAddr) + copyArgs, err := pageFaultCopyArgsForFault(regions, faultAddr) + if err != nil { + return 0, err } - return 0, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultAddr) + return copyArgs.srcOffset, nil } func pageFaultCopyArgsForFault(regions []GuestRegionUffdMapping, faultAddr uint64) (pageFaultCopyArgs, error) { diff --git a/memory/manager/uffd_socket.go b/memory/manager/uffd_socket.go index 03bcfc832..85a5ca1b7 100644 --- a/memory/manager/uffd_socket.go +++ b/memory/manager/uffd_socket.go @@ -79,6 +79,7 @@ func parseUnixRights(oob []byte) ([]int, error) { for i := range scms { rights, err := unix.ParseUnixRights(&scms[i]) if err != nil { + closeFDs(fds) return nil, err } fds = append(fds, rights...) diff --git a/memory/manager/uffd_socket_test.go b/memory/manager/uffd_socket_test.go index 7483443fc..23daaf914 100644 --- a/memory/manager/uffd_socket_test.go +++ b/memory/manager/uffd_socket_test.go @@ -61,6 +61,19 @@ func TestReceiveUffdMappingsAndFDInvalidJSON(t *testing.T) { } } +func TestReceiveUffdMappingsAndFDEmptyMappings(t *testing.T) { + sentFile := tempFileWithContent(t, "unused") + _, gotFile, err := receiveFromTestSocket(t, func(conn *net.UnixConn) error { + return writeUffdSocketPayload(conn, []byte("[]"), sentFile) + }) + if gotFile != nil { + _ = gotFile.Close() + } + if !errors.Is(err, errNoGuestRegionMappings) { + t.Fatalf("receiveUffdMappingsAndFD error = %v, want %v", err, errNoGuestRegionMappings) + } +} + func TestReceiveUffdMappingsAndFDMissingFD(t *testing.T) { body := validMappingsJSON(t) _, gotFile, err := receiveFromTestSocket(t, func(conn *net.UnixConn) error { From a04076919d653d010e57b8207773d5495c81e0c8 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Sat, 20 Jun 2026 18:30:15 +0800 Subject: [PATCH 04/21] Enable UPF snapshot load path --- ctriface/iface.go | 123 +++++++++++++++++++++-------- ctriface/iface_test.go | 5 -- go.mod | 3 +- go.sum | 6 +- memory/manager/manager.go | 38 ++++++++- memory/manager/snapshot_state.go | 45 ++++------- memory/manager/uffd_socket.go | 55 ++++++++++++- memory/manager/uffd_socket_test.go | 53 ++++++++----- vhive.go | 5 -- vhive_test.go | 5 -- 10 files changed, 230 insertions(+), 108 deletions(-) diff --git a/ctriface/iface.go b/ctriface/iface.go index 0ac07fefa..399fc9976 100644 --- a/ctriface/iface.go +++ b/ctriface/iface.go @@ -316,6 +316,12 @@ func (o *Orchestrator) StopSingleVM(ctx context.Context, vmID string) error { logger.WithError(err).Error("failed to stop firecracker-containerd VM") } + if o.GetUPFEnabled() { + if err := o.memoryManager.Deactivate(vmID); err != nil { + logger.WithError(err).Warn("failed to deactivate VM in memory manager") + } + } + if err := o.vmPool.Free(vmID); err != nil { logger.Error("failed to free VM from VM pool") return err @@ -493,6 +499,48 @@ func (o *Orchestrator) CreateSnapshot(ctx context.Context, vmID string, snap *sn return nil } +func logSnapshotLoadFailure(logger *log.Entry, snap *snapshotting.Snapshot, conf *proto.CreateVMRequest, err error) { + logger.WithError(err).Error("failed to load snapshot of the VM") + logger.Errorf("snapFilePath: %s, memFilePath: %s, containerSnapshotPath: %s", snap.GetSnapshotFilePath(), snap.GetMemFilePath(), conf.ContainerSnapshotPath) + + logDir := func(path string) { + files, readErr := os.ReadDir(filepath.Dir(path)) + if readErr != nil { + logger.Error(readErr) + return + } + + snapFiles := "" + for _, f := range files { + snapFiles += f.Name() + ", " + } + logger.Error(snapFiles) + } + + logDir(snap.GetSnapshotFilePath()) + logDir(conf.ContainerSnapshotPath) +} + +func waitForUnixSocket(socketPath string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + info, err := os.Lstat(socketPath) + if err == nil { + if info.Mode()&os.ModeSocket == 0 { + return fmt.Errorf("%s exists but is not a unix socket", socketPath) + } + return nil + } + if !os.IsNotExist(err) { + return err + } + if time.Now().After(deadline) { + return fmt.Errorf("timed out waiting for unix socket %s", socketPath) + } + time.Sleep(time.Millisecond) + } +} + // LoadSnapshot Loads a snapshot of a VM func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snapshotting.Snapshot) (_ *StartVMResponse, _ *metrics.Metric, retErr error) { var ( @@ -525,6 +573,7 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap conf.LoadSnapshot = true conf.SnapshotPath = snap.GetSnapshotFilePath() conf.MemFilePath = snap.GetMemFilePath() + uffdSock := filepath.Join(o.getVMBaseDir(vmID), "uffd.sock") if o.snapshotter == "devmapper" { if vm.Image, err = o.getImage(ctx, snap.GetImage()); err != nil { @@ -583,6 +632,25 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap } if o.GetUPFEnabled() { + conf.MemFilePath = "" + conf.MemBackend = &proto.MemoryBackend{ + BackendType: "Uffd", + BackendPath: uffdSock, + } + + if err := o.memoryManager.PrepareSnapshotLoad(manager.SnapshotStateCfg{ + VMID: vmID, + VMMStatePath: snap.GetSnapshotFilePath(), + GuestMemPath: snap.GetMemFilePath(), + WorkingSetPath: o.getWorkingSetFile(vmID), + InstanceSockAddr: uffdSock, + BaseDir: o.getVMBaseDir(vmID), + IsLazyMode: o.isLazyMode, + GuestMemSize: int(conf.MachineCfg.MemSizeMib) * 1024 * 1024, + }); err != nil { + return nil, nil, err + } + if err := o.memoryManager.FetchState(vmID); err != nil { return nil, nil, err } @@ -590,44 +658,35 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap tStart = time.Now() - go func() { - defer close(loadDone) + if o.GetUPFEnabled() { + activateDone := make(chan error, 1) + go func() { + activateDone <- o.memoryManager.Activate(vmID) + }() + + if err := waitForUnixSocket(uffdSock, time.Second); err != nil { + activateErr = <-activateDone + return nil, nil, multierror.Of(err, activateErr) + } if _, loadErr = o.fcClient.CreateVM(ctx, conf); loadErr != nil { - logger.Error("Failed to load snapshot of the VM: ", loadErr) - logger.Errorf("snapFilePath: %s, memFilePath: %s, containerSnapshotPath: %s", snap.GetSnapshotFilePath(), snap.GetMemFilePath(), conf.ContainerSnapshotPath) - files, err := os.ReadDir(filepath.Dir(snap.GetSnapshotFilePath())) - if err != nil { - logger.Error(err) - } - - snapFiles := "" - for _, f := range files { - snapFiles += f.Name() + ", " - } - - logger.Error(snapFiles) - - files, _ = os.ReadDir(filepath.Dir(conf.ContainerSnapshotPath)) - if err != nil { - logger.Error(err) - } - - snapFiles = "" - for _, f := range files { - snapFiles += f.Name() + ", " - } - logger.Error(snapFiles) + logSnapshotLoadFailure(logger, snap, conf, loadErr) } - }() - - if o.GetUPFEnabled() { - if activateErr = o.memoryManager.Activate(vmID); activateErr != nil { + activateErr = <-activateDone + if activateErr != nil { logger.Warn("Failed to activate VM in the memory manager", activateErr) } - } + } else { + go func() { + defer close(loadDone) - <-loadDone + if _, loadErr = o.fcClient.CreateVM(ctx, conf); loadErr != nil { + logSnapshotLoadFailure(logger, snap, conf, loadErr) + } + }() + + <-loadDone + } loadSnapshotMetric.MetricMap[metrics.LoadVMM] = metrics.ToUS(time.Since(tStart)) diff --git a/ctriface/iface_test.go b/ctriface/iface_test.go index cbf461187..6a7b0eb32 100644 --- a/ctriface/iface_test.go +++ b/ctriface/iface_test.go @@ -51,11 +51,6 @@ var ( func TestMain(m *testing.M) { flag.Parse() - if *isUPFEnabled { - log.Error("User-level page faults are temporarily disabled (gh-807)") - os.Exit(-1) - } - os.Exit(m.Run()) } diff --git a/go.mod b/go.mod index 150e1b96b..d4261c736 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/vhive-serverless/vhive go 1.26.2 replace ( - github.com/firecracker-microvm/firecracker-containerd => github.com/vhive-serverless/firecracker-containerd v0.0.0-20251220120737-a3dabeb332c7 + github.com/firecracker-microvm/firecracker-containerd => github.com/vhive-serverless/firecracker-containerd v0.0.0-20260304152437-f7f97e2dec10 github.com/vhive-serverless/vhive/examples/protobuf/helloworld => ./examples/protobuf/helloworld ) @@ -13,7 +13,6 @@ require ( github.com/containerd/stargz-snapshotter v0.11.3 github.com/davecgh/go-spew v1.1.1 github.com/firecracker-microvm/firecracker-containerd v0.0.0-00010101000000-000000000000 - github.com/ftrvxmtrx/fd v0.0.0-20150925145434-c6d800382fff github.com/go-multierror/multierror v1.0.2 github.com/golang/protobuf v1.5.4 github.com/google/nftables v0.3.0 diff --git a/go.sum b/go.sum index 5dd95ddfd..469723d32 100644 --- a/go.sum +++ b/go.sum @@ -385,8 +385,6 @@ github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ftrvxmtrx/fd v0.0.0-20150925145434-c6d800382fff h1:zk1wwii7uXmI0znwU+lqg+wFL9G5+vm5I+9rv2let60= -github.com/ftrvxmtrx/fd v0.0.0-20150925145434-c6d800382fff/go.mod h1:yUhRXHewUVJ1k89wHKP68xfzk7kwXUx/DV1nx4EBMbw= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= @@ -912,8 +910,8 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= -github.com/vhive-serverless/firecracker-containerd v0.0.0-20251220120737-a3dabeb332c7 h1:u+ZapSTjyKN+pDaLJu2YrmUddvzgyNy4JEWcwlwAyXI= -github.com/vhive-serverless/firecracker-containerd v0.0.0-20251220120737-a3dabeb332c7/go.mod h1:IlWnPBeebJFzQ0G12229J/hy9f4RhfdDTRC2+qAaXh0= +github.com/vhive-serverless/firecracker-containerd v0.0.0-20260304152437-f7f97e2dec10 h1:xLKB3s6Dn3yFME0yMBBcOP6Uvzfoc+HzT00wbZzrN7o= +github.com/vhive-serverless/firecracker-containerd v0.0.0-20260304152437-f7f97e2dec10/go.mod h1:XC5a/4PWbzipD5Ron745odZxoVy/J6d8xFldwTZJbSU= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= diff --git a/memory/manager/manager.go b/memory/manager/manager.go index 9d1e394bb..fc55550ac 100644 --- a/memory/manager/manager.go +++ b/memory/manager/manager.go @@ -90,6 +90,36 @@ func (m *MemoryManager) RegisterVM(cfg SnapshotStateCfg) error { return nil } +// PrepareSnapshotLoad creates or refreshes the state used to serve UFFD faults +// while loading a VM snapshot. +func (m *MemoryManager) PrepareSnapshotLoad(cfg SnapshotStateCfg) error { + m.Lock() + defer m.Unlock() + + vmID := cfg.VMID + if vmID == "" { + return errors.New("VMID is required") + } + + cfg.metricsModeOn = m.MetricsModeOn + nextState := NewSnapshotState(cfg) + + state, ok := m.instances[vmID] + if !ok { + m.instances[vmID] = nextState + return nil + } + if state.isActive { + return errors.New("failed to prepare snapshot load, VM still active") + } + if state.userFaultFD != nil { + _ = state.userFaultFD.Close() + } + + *state = *nextState + return nil +} + // DeregisterVM Deregisters a VM from the memory manager func (m *MemoryManager) DeregisterVM(vmID string) error { m.Lock() @@ -150,6 +180,9 @@ func (m *MemoryManager) Activate(vmID string) error { if err := state.getUFFD(); err != nil { logger.Error("Failed to get uffd") + if unmapErr := state.unmapGuestMemory(); unmapErr != nil { + logger.WithError(unmapErr).Error("Failed to munmap guest memory after getUFFD failure") + } return err } @@ -230,7 +263,10 @@ func (m *MemoryManager) Deactivate(vmID string) error { return errors.New("VM not activated") } - state.quitCh <- 0 + select { + case state.quitCh <- 0: + default: + } if err := state.unmapGuestMemory(); err != nil { logger.Error("Failed to munmap guest memory") return err diff --git a/memory/manager/snapshot_state.go b/memory/manager/snapshot_state.go index 1f6dc3247..0e7689752 100644 --- a/memory/manager/snapshot_state.go +++ b/memory/manager/snapshot_state.go @@ -28,11 +28,9 @@ package manager import "C" import ( - "context" "encoding/binary" "errors" "fmt" - "net" "os" "path/filepath" "sort" @@ -116,7 +114,7 @@ func (s *SnapshotState) setupStateOnActivate() { s.isActive = true s.isEverActivated = true s.firstPageFaultOnce = new(sync.Once) - s.quitCh = make(chan int) + s.quitCh = make(chan int, 1) if s.metricsModeOn { s.uniqueNum = 0 @@ -126,36 +124,16 @@ func (s *SnapshotState) setupStateOnActivate() { } func (s *SnapshotState) getUFFD() error { - var d net.Dialer - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - for { - c, err := d.DialContext(ctx, "unix", s.InstanceSockAddr) - if err != nil { - if ctx.Err() != nil { - log.Error("Failed to dial within the context timeout") - return err - } - time.Sleep(1 * time.Millisecond) - continue - } - - defer func() { _ = c.Close() }() - - sendfdConn := c.(*net.UnixConn) - - mappings, userFaultFD, err := receiveUffdMappingsAndFD(sendfdConn) - if err != nil { - log.Error("Failed to receive the uffd and guest memory mappings") - return err - } + mappings, userFaultFD, err := receiveUffdMappingsAndFDFromSocket(s.InstanceSockAddr) + if err != nil { + log.Error("Failed to receive the uffd and guest memory mappings") + return err + } - s.guestRegionMappings = mappings - s.userFaultFD = userFaultFD + s.guestRegionMappings = mappings + s.userFaultFD = userFaultFD - return nil - } + return nil } func (s *SnapshotState) processMetrics() { @@ -184,6 +162,7 @@ func (s *SnapshotState) mapGuestMemory() error { log.Errorf("Failed to open guest memory file: %v", err) return err } + defer func() { _ = fd.Close() }() s.guestMem, err = unix.Mmap(int(fd.Fd()), 0, s.GuestMemSize, unix.PROT_READ, unix.MAP_PRIVATE) if err != nil { @@ -246,6 +225,10 @@ func (s *SnapshotState) fetchState() error { return err } + if !s.IsLazyMode { + return nil + } + size := len(s.trace.trace) * os.Getpagesize() // O_DIRECT allows to fully leverage disk bandwidth by bypassing the OS page cache diff --git a/memory/manager/uffd_socket.go b/memory/manager/uffd_socket.go index 85a5ca1b7..728f10f4f 100644 --- a/memory/manager/uffd_socket.go +++ b/memory/manager/uffd_socket.go @@ -6,22 +6,57 @@ import ( "fmt" "net" "os" + "path/filepath" "time" "golang.org/x/sys/unix" ) const ( - uffdSocketPayloadSize = 64 * 1024 - uffdSocketFDLimit = 2 - uffdSocketReadTimeout = time.Second + uffdSocketPayloadSize = 64 * 1024 + uffdSocketFDLimit = 2 + uffdSocketReadTimeout = time.Second + uffdSocketAcceptTimeout = 30 * time.Second ) var ( errUnexpectedUffdFDCount = errors.New("expected exactly one uffd fd") errNoGuestRegionMappings = errors.New("no guest region mappings received") + errEmptyUffdSocketPath = errors.New("empty uffd socket path") ) +func receiveUffdMappingsAndFDFromSocket(socketPath string) ([]GuestRegionUffdMapping, *os.File, error) { + if socketPath == "" { + return nil, nil, errEmptyUffdSocketPath + } + if err := os.MkdirAll(filepath.Dir(socketPath), 0755); err != nil { + return nil, nil, err + } + if err := removeStaleUffdSocket(socketPath); err != nil { + return nil, nil, err + } + + addr := &net.UnixAddr{Name: socketPath, Net: "unix"} + listener, err := net.ListenUnix("unix", addr) + if err != nil { + return nil, nil, err + } + defer func() { _ = listener.Close() }() + defer func() { _ = os.Remove(socketPath) }() + + if err := listener.SetDeadline(time.Now().Add(uffdSocketAcceptTimeout)); err != nil { + return nil, nil, err + } + + conn, err := listener.AcceptUnix() + if err != nil { + return nil, nil, err + } + defer func() { _ = conn.Close() }() + + return receiveUffdMappingsAndFD(conn) +} + func receiveUffdMappingsAndFD(conn *net.UnixConn) ([]GuestRegionUffdMapping, *os.File, error) { if err := conn.SetReadDeadline(time.Now().Add(uffdSocketReadTimeout)); err != nil { return nil, nil, err @@ -93,3 +128,17 @@ func closeFDs(fds []int) { _ = unix.Close(receivedFD) } } + +func removeStaleUffdSocket(socketPath string) error { + info, err := os.Lstat(socketPath) + if err == nil { + if info.Mode()&os.ModeSocket == 0 { + return fmt.Errorf("refusing to remove non-socket uffd path %q", socketPath) + } + return os.Remove(socketPath) + } + if os.IsNotExist(err) { + return nil + } + return err +} diff --git a/memory/manager/uffd_socket_test.go b/memory/manager/uffd_socket_test.go index 23daaf914..de5c8f0d2 100644 --- a/memory/manager/uffd_socket_test.go +++ b/memory/manager/uffd_socket_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "reflect" "testing" + "time" "golang.org/x/sys/unix" ) @@ -116,36 +117,31 @@ func TestSnapshotStateGetUFFDStoresMappingsAndFD(t *testing.T) { sentFile := tempFileWithContent(t, "state fd payload") socketPath := filepath.Join(t.TempDir(), "uffd.sock") - listener, err := net.ListenUnix("unix", &net.UnixAddr{Name: socketPath, Net: "unix"}) - if err != nil { - t.Fatalf("net.ListenUnix returned error: %v", err) + state := &SnapshotState{ + SnapshotStateCfg: SnapshotStateCfg{ + InstanceSockAddr: socketPath, + }, } - defer func() { _ = listener.Close() }() - serverErrCh := make(chan error, 1) + stateErrCh := make(chan error, 1) go func() { - conn, err := listener.AcceptUnix() - if err != nil { - serverErrCh <- err - return - } - defer func() { _ = conn.Close() }() - serverErrCh <- writeUffdSocketPayload(conn, body, sentFile) + stateErrCh <- state.getUFFD() }() - state := &SnapshotState{ - SnapshotStateCfg: SnapshotStateCfg{ - InstanceSockAddr: socketPath, - }, + conn := dialUnixSocketWithRetry(t, socketPath) + if err := writeUffdSocketPayload(conn, body, sentFile); err != nil { + _ = conn.Close() + t.Fatalf("writeUffdSocketPayload returned error: %v", err) + } + if err := conn.Close(); err != nil { + t.Fatalf("conn.Close returned error: %v", err) } - if err := state.getUFFD(); err != nil { + + if err := <-stateErrCh; err != nil { t.Fatalf("getUFFD returned error: %v", err) } defer func() { _ = state.userFaultFD.Close() }() - if err := <-serverErrCh; err != nil { - t.Fatalf("test server returned error: %v", err) - } if !reflect.DeepEqual(state.guestRegionMappings, mappings) { t.Fatalf("guestRegionMappings = %+v, want %+v", state.guestRegionMappings, mappings) } @@ -158,6 +154,23 @@ func TestSnapshotStateGetUFFDStoresMappingsAndFD(t *testing.T) { } } +func dialUnixSocketWithRetry(t *testing.T, socketPath string) *net.UnixConn { + t.Helper() + + addr := &net.UnixAddr{Name: socketPath, Net: "unix"} + deadline := time.Now().Add(time.Second) + for { + conn, err := net.DialUnix("unix", nil, addr) + if err == nil { + return conn + } + if time.Now().After(deadline) { + t.Fatalf("net.DialUnix(%q) timed out: %v", socketPath, err) + } + time.Sleep(time.Millisecond) + } +} + func receiveFromTestSocket(t *testing.T, send func(*net.UnixConn) error) ([]GuestRegionUffdMapping, *os.File, error) { t.Helper() diff --git a/vhive.go b/vhive.go index 297a04af8..c1bddf843 100644 --- a/vhive.go +++ b/vhive.go @@ -93,11 +93,6 @@ func main() { return } - if *isUPFEnabled { - log.Error("User-level page faults are temporarily disabled (gh-807)") - return - } - if *isUPFEnabled && !*isSnapshotsEnabled { log.Error("User-level page faults are not supported without snapshots") return diff --git a/vhive_test.go b/vhive_test.go index a9160a32d..9c5450db9 100644 --- a/vhive_test.go +++ b/vhive_test.go @@ -65,11 +65,6 @@ func TestMain(m *testing.M) { flag.Parse() - if *isUPFEnabledTest { - log.Error("User-level page faults are temporarily disabled (gh-807)") - os.Exit(-1) - } - log.Infof("Orchestrator snapshots enabled: %t", *isSnapshotsEnabledTest) log.Infof("Orchestrator UPF enabled: %t", *isUPFEnabledTest) log.Infof("Orchestrator lazy serving mode enabled: %t", *isLazyModeTest) From 25b4bcccadc93b54ea52bd83866e345da8cdbbf1 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Sat, 20 Jun 2026 18:55:29 +0800 Subject: [PATCH 05/21] Fix UPF firecracker proto compatibility --- ctriface/proto_compat_test.go | 29 +++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- go.work.sum | 5 +++++ 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 ctriface/proto_compat_test.go diff --git a/ctriface/proto_compat_test.go b/ctriface/proto_compat_test.go new file mode 100644 index 000000000..692fd0f63 --- /dev/null +++ b/ctriface/proto_compat_test.go @@ -0,0 +1,29 @@ +package ctriface + +import ( + "testing" + + fcproto "github.com/firecracker-microvm/firecracker-containerd/proto" + "github.com/stretchr/testify/require" + googleproto "google.golang.org/protobuf/proto" +) + +func TestCreateVMRequestWithMemoryBackendMarshalsForTTRPC(t *testing.T) { + req := &fcproto.CreateVMRequest{ + VMID: "vm-with-upf", + LoadSnapshot: true, + MemBackend: &fcproto.MemoryBackend{ + BackendType: "Uffd", + BackendPath: "/tmp/vhive-upf.sock", + }, + } + + payload, err := googleproto.Marshal(req) + require.NoError(t, err) + require.NotEmpty(t, payload) + + var decoded fcproto.CreateVMRequest + require.NoError(t, googleproto.Unmarshal(payload, &decoded)) + require.Equal(t, "Uffd", decoded.GetMemBackend().GetBackendType()) + require.Equal(t, "/tmp/vhive-upf.sock", decoded.GetMemBackend().GetBackendPath()) +} diff --git a/go.mod b/go.mod index d4261c736..cfe5d3b17 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/vhive-serverless/vhive go 1.26.2 replace ( - github.com/firecracker-microvm/firecracker-containerd => github.com/vhive-serverless/firecracker-containerd v0.0.0-20260304152437-f7f97e2dec10 + github.com/firecracker-microvm/firecracker-containerd => github.com/vhive-serverless/firecracker-containerd v0.0.0-20260205055657-a35b52c279d9 github.com/vhive-serverless/vhive/examples/protobuf/helloworld => ./examples/protobuf/helloworld ) diff --git a/go.sum b/go.sum index 469723d32..d587269d0 100644 --- a/go.sum +++ b/go.sum @@ -910,8 +910,8 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= -github.com/vhive-serverless/firecracker-containerd v0.0.0-20260304152437-f7f97e2dec10 h1:xLKB3s6Dn3yFME0yMBBcOP6Uvzfoc+HzT00wbZzrN7o= -github.com/vhive-serverless/firecracker-containerd v0.0.0-20260304152437-f7f97e2dec10/go.mod h1:XC5a/4PWbzipD5Ron745odZxoVy/J6d8xFldwTZJbSU= +github.com/vhive-serverless/firecracker-containerd v0.0.0-20260205055657-a35b52c279d9 h1:Q2ybUII/hVdXKMA35KX54VLsrmydZ/HxxIKUxI6L31M= +github.com/vhive-serverless/firecracker-containerd v0.0.0-20260205055657-a35b52c279d9/go.mod h1:JDguJbk1e92EIfqyO7VRIeljZy3vw+JsJO62JGsl2i8= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= diff --git a/go.work.sum b/go.work.sum index 9a8897e9c..880debd3f 100644 --- a/go.work.sum +++ b/go.work.sum @@ -280,6 +280,7 @@ github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwys github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= +github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= @@ -288,6 +289,7 @@ github.com/containerd/btrfs/v2 v2.0.0/go.mod h1:swkD/7j9HApWpzl8OHfrHNxppPd9l44D github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= +github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0= github.com/containerd/containerd v1.6.23/go.mod h1:UrQOiyzrLi3n4aezYJbQH6Il+YzTvnHFbEuO3yfDrM4= @@ -303,6 +305,7 @@ github.com/containerd/ttrpc v1.1.1/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Ev github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= github.com/containerd/zfs v1.1.0/go.mod h1:oZF9wBnrnQjpWLaPKEinrx3TQ9a+W/RJO7Zb41d8YLE= github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= github.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEmnuFjskwo= github.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4= github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= @@ -400,6 +403,7 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -728,6 +732,7 @@ golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.164.0/go.mod h1:2OatzO7ZDQsoS7IFf3rvsE17/TldiU3F/zxFHeqUB5o= google.golang.org/api v0.166.0/go.mod h1:4FcBc686KFi7QI/U51/2GKKevfZMpM17sCdibqe/bSA= From 60b795f47dfb7478393393829a2be3105b9318a4 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Sun, 21 Jun 2026 11:10:23 +0800 Subject: [PATCH 06/21] Align UPF runtime binaries with reference stack --- bin/containerd-shim-aws-firecracker | 4 +- bin/firecracker | 4 +- bin/firecracker-containerd | 4 +- bin/jailer | 4 +- memory/manager/manager.go | 13 ++++++- memory/manager/snapshot_state.go | 59 ++++++++++++++++++++++------- 6 files changed, 64 insertions(+), 24 deletions(-) diff --git a/bin/containerd-shim-aws-firecracker b/bin/containerd-shim-aws-firecracker index d34106176..e2b219c37 100755 --- a/bin/containerd-shim-aws-firecracker +++ b/bin/containerd-shim-aws-firecracker @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:11cb33be5d00bcb8dcd61eb865f27afe19c36c2ff48a345559c1fe909f434cc4 -size 29662571 +oid sha256:81ff81fc3169fc4fafcd7e33588dc3ba528665b39f17cbe595e14dfd7690b118 +size 37271800 diff --git a/bin/firecracker b/bin/firecracker index 200d5ad4f..53d7006f1 100755 --- a/bin/firecracker +++ b/bin/firecracker @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:23a718b84dbe1a2b0978889657aa6fcb168819426a8603f1a09e4cf7e2a7c425 -size 10012352 +oid sha256:5fb7f47865abf24d55fddf1da7d96839a2a6c97396cbce64a531189329e9cb51 +size 3289608 diff --git a/bin/firecracker-containerd b/bin/firecracker-containerd index a6f1037f5..7cb01c83b 100755 --- a/bin/firecracker-containerd +++ b/bin/firecracker-containerd @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6d155d291d208b0e65b8edc9dc1c94e9135cb5dc2c0118fbc2d00cdbda5a2b1f -size 61495060 +oid sha256:1959d901c4a8a6bdf8394628d5c62c0d6eba23a976acfdf8f9173fc21bc26e68 +size 69041344 diff --git a/bin/jailer b/bin/jailer index 65bb271f0..958aa7a3f 100755 --- a/bin/jailer +++ b/bin/jailer @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d57d145e171bde170d1218dfc6ea705097760926a568f79f9c09af52bd671c6b -size 3461320 +oid sha256:7ff76f931a2b7542a959f08b69f21e092b0cfaffe314f3877c184f1452598498 +size 2289120 diff --git a/memory/manager/manager.go b/memory/manager/manager.go index fc55550ac..456d1726a 100644 --- a/memory/manager/manager.go +++ b/memory/manager/manager.go @@ -154,7 +154,7 @@ func (m *MemoryManager) Activate(vmID string) error { var ( ok bool state *SnapshotState - readyCh = make(chan int) + readyCh = make(chan error) ) m.Lock() @@ -190,7 +190,16 @@ func (m *MemoryManager) Activate(vmID string) error { go state.pollUserPageFaults(readyCh) - <-readyCh + if err := <-readyCh; err != nil { + logger.WithError(err).Error("Failed to start UFFD page fault polling") + if state.userFaultFD != nil { + _ = state.userFaultFD.Close() + } + if unmapErr := state.unmapGuestMemory(); unmapErr != nil { + logger.WithError(unmapErr).Error("Failed to munmap guest memory after UFFD poller failure") + } + return err + } return nil } diff --git a/memory/manager/snapshot_state.go b/memory/manager/snapshot_state.go index 0e7689752..bb48babc3 100644 --- a/memory/manager/snapshot_state.go +++ b/memory/manager/snapshot_state.go @@ -254,20 +254,21 @@ func (s *SnapshotState) fetchState() error { return nil } -func (s *SnapshotState) pollUserPageFaults(readyCh chan int) { +func (s *SnapshotState) pollUserPageFaults(readyCh chan error) { logger := log.WithFields(log.Fields{"vmID": s.VMID}) var events [1]syscall.EpollEvent if err := s.registerEpoller(); err != nil { - logger.Fatalf("register_epoller: %v", err) + readyCh <- err + return } logger.Debug("Starting polling loop") defer func() { _ = syscall.Close(s.epfd) }() - readyCh <- 0 + readyCh <- nil for { select { @@ -277,12 +278,19 @@ func (s *SnapshotState) pollUserPageFaults(readyCh chan int) { default: nevents, err := syscall.EpollWait(s.epfd, events[:], -1) if err != nil { - logger.Fatalf("epoll_wait: %v", err) - break + if errors.Is(err, syscall.EINTR) { + continue + } + if errors.Is(err, syscall.EBADF) { + logger.Debug("UFFD epoller was closed") + return + } + logger.WithError(err).Error("epoll_wait failed") + return } if nevents < 1 { - panic("Wrong number of events") + continue } for i := 0; i < nevents; i++ { @@ -293,26 +301,45 @@ func (s *SnapshotState) pollUserPageFaults(readyCh chan int) { stateFd := int(s.userFaultFD.Fd()) if fd != stateFd && stateFd != -1 { - logger.Fatalf("Received event from unknown fd") + logger.WithFields(log.Fields{ + "fd": fd, + "stateFd": stateFd, + }).Error("Received event from unknown fd") + return } goMsg := make([]byte, sizeOfUFFDMsg()) - if nread, err := syscall.Read(fd, goMsg); err != nil || nread != len(goMsg) { - if !errors.Is(err, syscall.EBADF) { - log.Fatalf("Read uffd_msg failed: %v", err) + nread, err := syscall.Read(fd, goMsg) + if err != nil { + if errors.Is(err, syscall.EINTR) || errors.Is(err, syscall.EAGAIN) { + continue + } + if errors.Is(err, syscall.EBADF) { + logger.Debug("UFFD fd was closed") + return } - break + logger.WithError(err).Error("Read uffd_msg failed") + return + } + if nread != len(goMsg) { + logger.WithFields(log.Fields{ + "read": nread, + "want": len(goMsg), + }).Error("Read incomplete uffd_msg") + return } if event := uint8(goMsg[0]); event != uffdPageFault() { - log.Fatal("Received wrong event type") + logger.WithField("event", event).Warn("Ignoring unsupported UFFD event") + continue } address := binary.LittleEndian.Uint64(goMsg[16:]) if err := s.servePageFault(fd, address); err != nil { - log.Fatalf("Failed to serve page fault") + logger.WithError(err).WithField("address", fmt.Sprintf("%#x", address)).Error("Failed to serve page fault") + return } } } @@ -345,6 +372,7 @@ func (s *SnapshotState) registerEpoller() error { fdInt, &event, ); err != nil { + _ = syscall.Close(s.epfd) logger.Errorf("Failed to subscribe VM %v", err) return err } @@ -470,6 +498,9 @@ func installRegionBytes(fd int, src, dst, mode, length uint64) error { err := ioctl(uintptr(fd), int(C.const_UFFDIO_COPY), unsafe.Pointer(&cUC)) if err != nil { + if errors.Is(err, unix.EEXIST) { + return nil + } return err } @@ -498,7 +529,7 @@ func ioctl(fd uintptr, request int, argp unsafe.Pointer) error { uintptr(argp), ) if errno != 0 { - return os.NewSyscallError("ioctl", fmt.Errorf("%d", int(errno))) + return os.NewSyscallError("ioctl", errno) } return nil From 77885298a7f4851a8f08c4b634b600d51ec83753 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Sun, 21 Jun 2026 11:21:07 +0800 Subject: [PATCH 07/21] Align Firecracker 1.13 guest boot config --- ctriface/iface.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctriface/iface.go b/ctriface/iface.go index 399fc9976..39aeba1e6 100644 --- a/ctriface/iface.go +++ b/ctriface/iface.go @@ -365,7 +365,7 @@ func getK8sDNS() []string { } func (o *Orchestrator) getVMConfig(vm *misc.VM) *proto.CreateVMRequest { - kernelArgs := "ro noapic reboot=k panic=1 pci=off nomodules systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init tsc=reliable quiet 8250.nr_uarts=0 ipv6.disable=1" + kernelArgs := "ro noapic reboot=k panic=1 acpi=off pci=off nomodules systemd.log_color=false systemd.journald.forward_to_console systemd.unit=firecracker.target init=/sbin/overlay-init tsc=reliable quiet ipv6.disable=1 console=ttyS0" return &proto.CreateVMRequest{ VMID: vm.ID, @@ -373,7 +373,7 @@ func (o *Orchestrator) getVMConfig(vm *misc.VM) *proto.CreateVMRequest { KernelArgs: kernelArgs, MachineCfg: &proto.FirecrackerMachineConfiguration{ VcpuCount: 1, - MemSizeMib: 256, + MemSizeMib: 512, }, NetworkInterfaces: []*proto.FirecrackerNetworkInterface{{ AllowMMDS: true, From 29253b16e11e03de5e97f2f91761e6a811ce4a6f Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Tue, 23 Jun 2026 22:46:17 +0800 Subject: [PATCH 08/21] Refactor UPF UFFD snapshot handling --- ctriface/iface.go | 67 +-- ctriface/orch.go | 1 - ctriface/orch_options.go | 5 +- memory/manager/manager.go | 91 +--- memory/manager/manager_test.go | 303 ++++++++----- memory/manager/snapshot_state.go | 418 ++++++++++-------- ..._socket_test.go => snapshot_state_test.go} | 152 ++++++- memory/manager/trace.go | 210 --------- memory/manager/uffd_mapping.go | 104 ----- memory/manager/uffd_mapping_test.go | 242 ---------- memory/manager/uffd_socket.go | 144 ------ memory/manager/user_page_faults.h | 35 -- 12 files changed, 597 insertions(+), 1175 deletions(-) rename memory/manager/{uffd_socket_test.go => snapshot_state_test.go} (65%) delete mode 100644 memory/manager/trace.go delete mode 100644 memory/manager/uffd_mapping.go delete mode 100644 memory/manager/uffd_mapping_test.go delete mode 100644 memory/manager/uffd_socket.go diff --git a/ctriface/iface.go b/ctriface/iface.go index 39aeba1e6..c0f8b8e6e 100644 --- a/ctriface/iface.go +++ b/ctriface/iface.go @@ -248,15 +248,11 @@ func (o *Orchestrator) StartVMWithEnvironment(ctx context.Context, vmID, imageNa logger.Debug("Registering VM with the memory manager") stateCfg := manager.SnapshotStateCfg{ - VMID: vmID, - GuestMemPath: o.getMemoryFile(vmID), - BaseDir: o.getVMBaseDir(vmID), - GuestMemSize: int(conf.MachineCfg.MemSizeMib) * 1024 * 1024, - IsLazyMode: o.isLazyMode, - VMMStatePath: o.getSnapshotFile(vmID), - WorkingSetPath: o.getWorkingSetFile(vmID), - // FIXME (gh-807) - //InstanceSockAddr: resp.UPFSockPath, + VMID: vmID, + GuestMemPath: o.getMemoryFile(vmID), + BaseDir: o.getVMBaseDir(vmID), + GuestMemSize: int(conf.MachineCfg.MemSizeMib) * 1024 * 1024, + VMMStatePath: o.getSnapshotFile(vmID), } if err := o.memoryManager.RegisterVM(stateCfg); err != nil { return nil, nil, errors.Wrap(err, "failed to register VM with memory manager") @@ -500,45 +496,13 @@ func (o *Orchestrator) CreateSnapshot(ctx context.Context, vmID string, snap *sn } func logSnapshotLoadFailure(logger *log.Entry, snap *snapshotting.Snapshot, conf *proto.CreateVMRequest, err error) { - logger.WithError(err).Error("failed to load snapshot of the VM") - logger.Errorf("snapFilePath: %s, memFilePath: %s, containerSnapshotPath: %s", snap.GetSnapshotFilePath(), snap.GetMemFilePath(), conf.ContainerSnapshotPath) - - logDir := func(path string) { - files, readErr := os.ReadDir(filepath.Dir(path)) - if readErr != nil { - logger.Error(readErr) - return - } - - snapFiles := "" - for _, f := range files { - snapFiles += f.Name() + ", " - } - logger.Error(snapFiles) - } - - logDir(snap.GetSnapshotFilePath()) - logDir(conf.ContainerSnapshotPath) -} - -func waitForUnixSocket(socketPath string, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - for { - info, err := os.Lstat(socketPath) - if err == nil { - if info.Mode()&os.ModeSocket == 0 { - return fmt.Errorf("%s exists but is not a unix socket", socketPath) - } - return nil - } - if !os.IsNotExist(err) { - return err - } - if time.Now().After(deadline) { - return fmt.Errorf("timed out waiting for unix socket %s", socketPath) - } - time.Sleep(time.Millisecond) - } + logger.WithError(err).WithFields(log.Fields{ + "snapFilePath": snap.GetSnapshotFilePath(), + "memFilePath": snap.GetMemFilePath(), + "containerSnapshotPath": conf.ContainerSnapshotPath, + "memoryBackendType": conf.GetMemBackend().GetBackendType(), + "memoryBackendPath": conf.GetMemBackend().GetBackendPath(), + }).Error("failed to load snapshot of the VM") } // LoadSnapshot Loads a snapshot of a VM @@ -642,10 +606,8 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap VMID: vmID, VMMStatePath: snap.GetSnapshotFilePath(), GuestMemPath: snap.GetMemFilePath(), - WorkingSetPath: o.getWorkingSetFile(vmID), InstanceSockAddr: uffdSock, BaseDir: o.getVMBaseDir(vmID), - IsLazyMode: o.isLazyMode, GuestMemSize: int(conf.MachineCfg.MemSizeMib) * 1024 * 1024, }); err != nil { return nil, nil, err @@ -660,11 +622,12 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap if o.GetUPFEnabled() { activateDone := make(chan error, 1) + socketReady := make(chan error, 1) go func() { - activateDone <- o.memoryManager.Activate(vmID) + activateDone <- o.memoryManager.ActivateWithSocketReady(vmID, socketReady) }() - if err := waitForUnixSocket(uffdSock, time.Second); err != nil { + if err := <-socketReady; err != nil { activateErr = <-activateDone return nil, nil, multierror.Of(err, activateErr) } diff --git a/ctriface/orch.go b/ctriface/orch.go index b6ec2be04..6eadd10eb 100644 --- a/ctriface/orch.go +++ b/ctriface/orch.go @@ -100,7 +100,6 @@ type Orchestrator struct { // store *skv.KVStore snapshotsEnabled bool isUPFEnabled bool - isLazyMode bool snapshotsDir string isMetricsMode bool netPoolSize int diff --git a/ctriface/orch_options.go b/ctriface/orch_options.go index 321962f8d..70143f75d 100644 --- a/ctriface/orch_options.go +++ b/ctriface/orch_options.go @@ -62,12 +62,9 @@ func WithSnapshotsDir(snapshotsDir string) OrchestratorOption { } } -// WithLazyMode Sets the lazy paging mode on (or off), -// where all guest memory pages are brought on demand. -// Only works if snapshots are enabled +// WithLazyMode is kept for compatibility with legacy callers. func WithLazyMode(isLazyMode bool) OrchestratorOption { return func(o *Orchestrator) { - o.isLazyMode = isLazyMode } } diff --git a/memory/manager/manager.go b/memory/manager/manager.go index 456d1726a..3695d8520 100644 --- a/memory/manager/manager.go +++ b/memory/manager/manager.go @@ -27,9 +27,7 @@ import ( "errors" "fmt" "os" - "strconv" "sync" - "time" "github.com/vhive-serverless/vhive/metrics" "gonum.org/v1/gonum/stat" @@ -39,8 +37,6 @@ import ( const ( serveUniqueMetric = "ServeUnique" - installWSMetric = "InstallWS" - fetchStateMetric = "FetchState" ) // MemoryManagerCfg Global config of the manager @@ -147,6 +143,16 @@ func (m *MemoryManager) DeregisterVM(vmID string) error { // Activate Creates an epoller to serve page faults for the VM func (m *MemoryManager) Activate(vmID string) error { + return m.activate(vmID, nil) +} + +// ActivateWithSocketReady is like Activate, but reports when the UFFD socket +// listener is ready for Firecracker to connect. +func (m *MemoryManager) ActivateWithSocketReady(vmID string, socketReadyCh chan<- error) error { + return m.activate(vmID, socketReadyCh) +} + +func (m *MemoryManager) activate(vmID string, socketReadyCh chan<- error) error { logger := log.WithFields(log.Fields{"vmID": vmID}) logger.Debug("Activating instance in the memory manager") @@ -175,10 +181,11 @@ func (m *MemoryManager) Activate(vmID string) error { if err := state.mapGuestMemory(); err != nil { logger.Error("Failed to map guest memory") + notifySocketReady(socketReadyCh, err) return err } - if err := state.getUFFD(); err != nil { + if err := state.getUFFD(socketReadyCh); err != nil { logger.Error("Failed to get uffd") if unmapErr := state.unmapGuestMemory(); unmapErr != nil { logger.WithError(unmapErr).Error("Failed to munmap guest memory after getUFFD failure") @@ -204,17 +211,15 @@ func (m *MemoryManager) Activate(vmID string) error { return nil } -// FetchState Fetches the working set file (or the whole guest memory) and the VMM state file +// FetchState verifies that snapshot state needed by the memory manager exists. func (m *MemoryManager) FetchState(vmID string) error { logger := log.WithFields(log.Fields{"vmID": vmID}) - logger.Debug("Activating instance in the memory manager") + logger.Debug("Fetching state in the memory manager") var ( - ok bool - state *SnapshotState - tStart time.Time - err error + ok bool + state *SnapshotState ) m.Lock() @@ -228,17 +233,7 @@ func (m *MemoryManager) FetchState(vmID string) error { m.Unlock() - if state.isRecordReady && !state.IsLazyMode { - if state.metricsModeOn { - tStart = time.Now() - } - err = state.fetchState() - if state.metricsModeOn { - state.currentMetric.MetricMap[fetchStateMetric] = metrics.ToUS(time.Since(tStart)) - } - } - - return err + return state.fetchState() } // Deactivate Removes the epoller which serves page faults for the VM @@ -283,12 +278,10 @@ func (m *MemoryManager) Deactivate(vmID string) error { state.processMetrics() - defer func() { _ = state.userFaultFD.Close() }() - if !state.isRecordReady && !state.IsLazyMode { - state.trace.ProcessRecord(state.GuestMemPath, state.WorkingSetPath) + if state.userFaultFD != nil { + defer func() { _ = state.userFaultFD.Close() }() } - state.isRecordReady = true state.isActive = false return nil @@ -326,11 +319,7 @@ func (m *MemoryManager) DumpUPFPageStats(vmID, functionName, metricsOutFilePath return errors.New("metrics mode is not on") } - if state.IsLazyMode { - statHeader, stats = getLazyHeaderStats(state, functionName) - } else { - statHeader, stats = getRecRepHeaderStats(state, functionName) - } + statHeader, stats = getUPFHeaderStats(state, functionName) return writeUPFPageStats(metricsOutFilePath, statHeader, stats) } @@ -396,42 +385,10 @@ func (m *MemoryManager) GetUPFLatencyStats(vmID string) ([]*metrics.Metric, erro return state.latencyMetrics, nil } -func getLazyHeaderStats(state *SnapshotState, functionName string) ([]string, []string) { - header := []string{ - "FuncName", - "RecPages", - "RepPages", - "StdDev", - "Reused", - "StdDev", - "Unique", - "StdDev", - } - - uniqueMean, uniqueStd := stat.MeanStdDev(state.uniquePFServed, nil) - totalMean, totalStd := stat.MeanStdDev(state.totalPFServed, nil) - reusedMean, reusedStd := stat.MeanStdDev(state.reusedPFServed, nil) - - stats := []string{ - functionName, - strconv.Itoa(len(state.trace.trace)), // number of records (i.e., offsets) - strconv.Itoa(int(totalMean)), // number of pages served - fmt.Sprintf("%.1f", totalStd), - strconv.Itoa(int(reusedMean)), // number of pages found in the trace - fmt.Sprintf("%.1f", reusedStd), - strconv.Itoa(int(uniqueMean)), // number of pages not found in the trace - fmt.Sprintf("%.1f", uniqueStd), - } - - return header, stats -} - -func getRecRepHeaderStats(state *SnapshotState, functionName string) ([]string, []string) { +func getUPFHeaderStats(state *SnapshotState, functionName string) ([]string, []string) { header := []string{ "FuncName", - "RecPages", - "RecRegions", - "Unique", + "ServedPages", "StdDev", } @@ -439,9 +396,7 @@ func getRecRepHeaderStats(state *SnapshotState, functionName string) ([]string, stats := []string{ functionName, - strconv.Itoa(len(state.trace.trace)), // number of records (i.e., offsets) - strconv.Itoa(len(state.trace.regions)), // number of contiguous regions in the trace - strconv.Itoa(int(uniqueMean)), // number of pages not found in the trace + fmt.Sprintf("%.0f", uniqueMean), fmt.Sprintf("%.1f", uniqueStd), } diff --git a/memory/manager/manager_test.go b/memory/manager/manager_test.go index 6559d9e26..abb38a060 100644 --- a/memory/manager/manager_test.go +++ b/memory/manager/manager_test.go @@ -20,150 +20,222 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -//nolint:unused package manager import ( + "encoding/binary" + "encoding/json" + "errors" "os" + "path/filepath" + "reflect" + "testing" + "time" - log "github.com/sirupsen/logrus" - - "errors" + "golang.org/x/sys/unix" ) -/* -func TestSingleClient(t *testing.T) { - log.SetFormatter(&log.TextFormatter{ - TimestampFormat: ctrdlog.RFC3339NanoFixed, - FullTimestamp: true, - }) - var ( - uffd int - region []byte - regionSize int = 4 * os.Getpagesize() - uffdFileName string = "/tmp/uffd_file.file" - guestMemoryPath = "/tmp/guest_mem" - vmID string = "1" - ) - - log.SetLevel(log.DebugLevel) - - prepareGuestMemoryFile(guestMemoryPath, regionSize) - - region, err := unix.Mmap(-1, 0, regionSize, unix.PROT_READ, unix.MAP_PRIVATE|unix.MAP_ANONYMOUS) +func TestPrepareGuestMemoryFileAndValidateGuestMemory(t *testing.T) { + guestMemPath := filepath.Join(t.TempDir(), "guest_mem") + guestMemSize := 4 * os.Getpagesize() + + prepareGuestMemoryFile(t, guestMemPath, guestMemSize) + + guestMem, err := os.ReadFile(guestMemPath) if err != nil { - log.Errorf("Failed to mmap: %v", err) + t.Fatalf("os.ReadFile returned error: %v", err) + } + if err := validateGuestMemory(guestMem); err != nil { + t.Fatalf("validateGuestMemory returned error: %v", err) } - defer unix.Munmap(region) - - uffd = registerForUpf(region, uint64(regionSize)) + guestMem[os.Getpagesize()] = 0 + if err := validateGuestMemory(guestMem); err == nil { + t.Fatal("validateGuestMemory succeeded for corrupted guest memory") + } +} - uffdFile := os.NewFile(uintptr(uffd), uffdFileName) +func TestMemoryManagerRegisterFetchPrepareDeregister(t *testing.T) { + baseDir := t.TempDir() + vmID := "vm-register" + guestMemPath := filepath.Join(baseDir, "guest_mem") + vmmStatePath := filepath.Join(baseDir, "state") - managerCfg := MemoryManagerCfg{} - manager := NewMemoryManager(managerCfg) + prepareGuestMemoryFile(t, guestMemPath, os.Getpagesize()) + writeTestFile(t, vmmStatePath, "state") - stateCfg := SnapshotStateCfg{ + manager := NewMemoryManager(MemoryManagerCfg{}) + cfg := SnapshotStateCfg{ VMID: vmID, - BaseDir: "/tmp/snap_base", - GuestMemPath: guestMemoryPath, - GuestMemSize: regionSize, + BaseDir: baseDir, + VMMStatePath: vmmStatePath, + GuestMemPath: guestMemPath, + GuestMemSize: os.Getpagesize(), } - err = manager.RegisterVM(stateCfg) - require.NoError(t, err, "Failed to register VM") - - err = manager.Activate(vmID, uffdFile) - require.NoError(t, err, "Failed to add VM") + if err := manager.RegisterVM(cfg); err != nil { + t.Fatalf("RegisterVM returned error: %v", err) + } + if err := manager.RegisterVM(cfg); err == nil { + t.Fatal("RegisterVM succeeded for duplicate VM") + } + if err := manager.FetchState(vmID); err != nil { + t.Fatalf("FetchState returned error: %v", err) + } - err = validateGuestMemory(region) - require.NoError(t, err, "Failed to validate guest memory") + nextVMMStatePath := filepath.Join(baseDir, "next_state") + writeTestFile(t, nextVMMStatePath, "next-state") + nextCfg := cfg + nextCfg.VMMStatePath = nextVMMStatePath - err = manager.Deactivate(vmID) - require.NoError(t, err, "Failed to remove intance") + if err := manager.PrepareSnapshotLoad(nextCfg); err != nil { + t.Fatalf("PrepareSnapshotLoad returned error: %v", err) + } + if got := manager.instances[vmID].VMMStatePath; got != nextVMMStatePath { + t.Fatalf("PrepareSnapshotLoad VMMStatePath = %q, want %q", got, nextVMMStatePath) + } - err = manager.DeregisterVM(vmID) - require.NoError(t, err, "Failed to deregister vm") + if err := manager.DeregisterVM(vmID); err != nil { + t.Fatalf("DeregisterVM returned error: %v", err) + } + if err := manager.DeregisterVM(vmID); err == nil { + t.Fatal("DeregisterVM succeeded for unregistered VM") + } } -func TestParallelClients(t *testing.T) { - numParallel := 1000 +func TestMemoryManagerActivateReceivesFirecrackerMappings(t *testing.T) { + baseDir := t.TempDir() + vmID := "vm-activate" + guestMemPath := filepath.Join(baseDir, "guest_mem") + socketPath := filepath.Join(baseDir, "uffd.sock") + guestMemSize := 2 * os.Getpagesize() + + prepareGuestMemoryFile(t, guestMemPath, guestMemSize) + + manager := NewMemoryManager(MemoryManagerCfg{}) + cfg := SnapshotStateCfg{ + VMID: vmID, + BaseDir: baseDir, + GuestMemPath: guestMemPath, + GuestMemSize: guestMemSize, + InstanceSockAddr: socketPath, + } + if err := manager.RegisterVM(cfg); err != nil { + t.Fatalf("RegisterVM returned error: %v", err) + } - log.SetFormatter(&log.TextFormatter{ - TimestampFormat: ctrdlog.RFC3339NanoFixed, - FullTimestamp: true, - }) - var ( - regionSize int = 4 * os.Getpagesize() - err error - ) + mappings := []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: uint64(guestMemSize), + PageSize: uint64(os.Getpagesize()), + }} + body, err := json.Marshal(mappings) + if err != nil { + t.Fatalf("json.Marshal returned error: %v", err) + } - log.SetLevel(log.DebugLevel) + uffdStandIn := testEventFD(t) + socketReadyCh := make(chan error, 1) + activateErrCh := make(chan error, 1) + go func() { + activateErrCh <- manager.ActivateWithSocketReady(vmID, socketReadyCh) + }() - clients := make(map[int]*upfClient) + if err := receiveSocketReady(t, socketReadyCh); err != nil { + t.Fatalf("ActivateWithSocketReady failed before socket accept: %v", err) + } - for i := 0; i < numParallel; i++ { - vmID := fmt.Sprintf("%d", i) - guestMemoryPath := "/tmp/guest_mem_" + vmID + conn := dialUnixSocketWithRetry(t, socketPath) + if err := writeUffdSocketPayload(conn, body, uffdStandIn); err != nil { + _ = conn.Close() + t.Fatalf("writeUffdSocketPayload returned error: %v", err) + } + if err := conn.Close(); err != nil { + t.Fatalf("conn.Close returned error: %v", err) + } - prepareGuestMemoryFile(guestMemoryPath, regionSize) + if err := receiveActivateResult(t, activateErrCh); err != nil { + t.Fatalf("ActivateWithSocketReady returned error: %v", err) + } - region, err := unix.Mmap(-1, 0, regionSize, unix.PROT_READ, unix.MAP_PRIVATE|unix.MAP_ANONYMOUS) - if err != nil { - log.Errorf("Failed to mmap: %v", err) - } - defer unix.Munmap(region) + state := manager.instances[vmID] + if !state.isActive { + t.Fatal("state is not active after ActivateWithSocketReady") + } + if !reflect.DeepEqual(state.guestRegionMappings, mappings) { + t.Fatalf("guestRegionMappings = %+v, want %+v", state.guestRegionMappings, mappings) + } + if err := validateGuestMemory(state.guestMem); err != nil { + t.Fatalf("validateGuestMemory mapped memory returned error: %v", err) + } - uffd := registerForUpf(region, uint64(regionSize)) - uffdFileName := fmt.Sprintf("file_%s", vmID) - uffdFile := os.NewFile(uintptr(uffd), uffdFileName) + signalEventFD(t, uffdStandIn) - clients[i] = initClient(uffd, region, uffdFileName, guestMemoryPath, vmID, uffdFile) + if err := manager.Deactivate(vmID); err != nil { + t.Fatalf("Deactivate returned error: %v", err) + } + if err := manager.DeregisterVM(vmID); err != nil { + t.Fatalf("DeregisterVM returned error: %v", err) } +} - managerCfg := MemoryManagerCfg{} - manager := NewMemoryManager(managerCfg) +func receiveSocketReady(t *testing.T, readyCh <-chan error) error { + t.Helper() - var wg sync.WaitGroup + select { + case err := <-readyCh: + return err + case <-time.After(time.Second): + t.Fatal("timed out waiting for UFFD socket readiness") + } - for i := 0; i < numParallel; i++ { - c := clients[i] - stateCfg := SnapshotStateCfg{ - VMID: c.vmID, - BaseDir: "/tmp/snap_base", - GuestMemPath: c.guestMemoryPath, - GuestMemSize: regionSize, - } + return nil +} - wg.Add(1) +func receiveActivateResult(t *testing.T, activateErrCh <-chan error) error { + t.Helper() - go func() { - defer wg.Done() + select { + case err := <-activateErrCh: + return err + case <-time.After(time.Second): + t.Fatal("timed out waiting for ActivateWithSocketReady") + } - err = manager.RegisterVM(stateCfg) - require.NoError(t, err, "Failed to register VM") + return nil +} - err = manager.Activate(c.vmID, c.uffdFile) - require.NoError(t, err, "Failed to add VM") +func testEventFD(t *testing.T) *os.File { + t.Helper() - err = validateGuestMemory(c.region) - require.NoError(t, err, "Failed to validate guest memory") + fd, err := unix.Eventfd(0, unix.EFD_CLOEXEC|unix.EFD_NONBLOCK) + if err != nil { + t.Fatalf("unix.Eventfd returned error: %v", err) + } + + file := os.NewFile(uintptr(fd), "test-eventfd") + if file == nil { + _ = unix.Close(fd) + t.Fatal("os.NewFile returned nil") + } + t.Cleanup(func() { _ = file.Close() }) - err = manager.Deactivate(c.vmID) - require.NoError(t, err, "Failed to remove intance") + return file +} - err = manager.DeregisterVM(c.vmID) - require.NoError(t, err, "Failed to deregister vm") - }() +func signalEventFD(t *testing.T, file *os.File) { + t.Helper() + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], 1) + if _, err := unix.Write(int(file.Fd()), buf[:]); err != nil { + t.Fatalf("unix.Write(eventfd) returned error: %v", err) } - - wg.Wait() } -*/ -func prepareGuestMemoryFile(guestFileName string, size int) { +func prepareGuestMemoryFile(t *testing.T, guestFileName string, size int) { + t.Helper() + toWrite := make([]byte, size) pages := size / os.Getpagesize() for i := 0; i < pages; i++ { @@ -172,40 +244,27 @@ func prepareGuestMemoryFile(guestFileName string, size int) { } } - err := os.WriteFile(guestFileName, toWrite, 0777) - if err != nil { - panic(err) + if err := os.WriteFile(guestFileName, toWrite, 0600); err != nil { + t.Fatalf("os.WriteFile returned error: %v", err) } } func validateGuestMemory(guestMem []byte) error { pages := len(guestMem) / os.Getpagesize() for i := 0; i < pages; i++ { - log.Debugf("Validating page %d's contents...\n", i) j := os.Getpagesize() * i if guestMem[j] != byte(48+i) { - return errors.New("Incorrect guest memory") + return errors.New("incorrect guest memory") } } - return nil -} -type upfClient struct { - uffd int - region []byte - uffdFileName, guestMemoryPath, vmID string - uffdFile *os.File + return nil } -func initClient(uffd int, region []byte, uffdFileName, guestMemoryPath, vmID string, uffdFile *os.File) *upfClient { - c := new(upfClient) - - c.uffd = uffd - c.region = region - c.uffdFileName = uffdFileName - c.guestMemoryPath = guestMemoryPath - c.vmID = vmID - c.uffdFile = uffdFile +func writeTestFile(t *testing.T, path, content string) { + t.Helper() - return c + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatalf("os.WriteFile returned error: %v", err) + } } diff --git a/memory/manager/snapshot_state.go b/memory/manager/snapshot_state.go index bb48babc3..844f0da2c 100644 --- a/memory/manager/snapshot_state.go +++ b/memory/manager/snapshot_state.go @@ -29,12 +29,13 @@ import "C" import ( "encoding/binary" + "encoding/json" "errors" "fmt" + "math" + "net" "os" "path/filepath" - "sort" - "sync" "syscall" "time" @@ -46,16 +47,103 @@ import ( "unsafe" ) +const ( + uffdSocketPayloadSize = 64 * 1024 + uffdSocketFDLimit = 2 + uffdSocketReadTimeout = time.Second + uffdSocketAcceptTimeout = 30 * time.Second +) + +var ( + errInvalidGuestRegionPageSize = errors.New("guest region page size must be non-zero") + errGuestRegionNotFound = errors.New("fault address is outside guest memory mappings") + errUnexpectedUffdFDCount = errors.New("expected exactly one uffd fd") + errNoGuestRegionMappings = errors.New("no guest region mappings received") + errEmptyUffdSocketPath = errors.New("empty uffd socket path") +) + +// GuestRegionUffdMapping describes Firecracker's UFFD guest memory mapping. +type GuestRegionUffdMapping struct { + BaseHostVirtAddr uint64 `json:"base_host_virt_addr"` + Size uint64 `json:"size"` + Offset uint64 `json:"offset"` + PageSize uint64 `json:"page_size"` +} + +type pageFaultCopyArgs struct { + srcOffset uint64 + dstAddr uint64 + copyLen uint64 + copyMode uint64 +} + +func pageAlignFaultAddress(faultAddr uint64, region GuestRegionUffdMapping) (uint64, error) { + if region.PageSize == 0 { + return 0, errInvalidGuestRegionPageSize + } + + return faultAddr - faultAddr%region.PageSize, nil +} + +func guestMemoryOffsetForFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) (uint64, error) { + if region.PageSize == 0 { + return 0, errInvalidGuestRegionPageSize + } + if !regionContainsFaultPage(region, faultPageAddr) { + return 0, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultPageAddr) + } + + regionOffset := faultPageAddr - region.BaseHostVirtAddr + if region.Offset > math.MaxUint64-regionOffset { + return 0, fmt.Errorf("guest memory offset overflow for fault address %#x", faultPageAddr) + } + + return region.Offset + regionOffset, nil +} + +func pageFaultCopyArgsForFault(regions []GuestRegionUffdMapping, faultAddr uint64) (pageFaultCopyArgs, error) { + for _, region := range regions { + faultPageAddr, err := pageAlignFaultAddress(faultAddr, region) + if err != nil { + return pageFaultCopyArgs{}, err + } + if !regionContainsFaultPage(region, faultPageAddr) { + continue + } + + srcOffset, err := guestMemoryOffsetForFaultPage(region, faultPageAddr) + if err != nil { + return pageFaultCopyArgs{}, err + } + + return pageFaultCopyArgs{ + srcOffset: srcOffset, + dstAddr: faultPageAddr, + copyLen: region.PageSize, + copyMode: 0, + }, nil + } + + return pageFaultCopyArgs{}, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultAddr) +} + +func regionContainsFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) bool { + if region.Size == 0 || faultPageAddr < region.BaseHostVirtAddr { + return false + } + + return faultPageAddr-region.BaseHostVirtAddr < region.Size +} + // SnapshotStateCfg Config to initialize SnapshotState type SnapshotStateCfg struct { VMID string - VMMStatePath, GuestMemPath, WorkingSetPath string + VMMStatePath, GuestMemPath string InstanceSockAddr string BaseDir string // base directory for the instance MetricsPath string // path to csv file where the metrics should be stored - IsLazyMode bool GuestMemSize int metricsModeOn bool } @@ -64,11 +152,8 @@ type SnapshotStateCfg struct { // of the VM. type SnapshotState struct { SnapshotStateCfg - firstPageFaultOnce *sync.Once // to initialize the start virtual address and replay - startAddress uint64 userFaultFD *os.File guestRegionMappings []GuestRegionUffdMapping - trace *Trace epfd int quitCh chan int @@ -78,18 +163,12 @@ type SnapshotState struct { // for sanity checking on deactivate/activate isActive bool - isRecordReady bool - - guestMem []byte - workingSet []byte + guestMem []byte // Stats - totalPFServed []float64 uniquePFServed []float64 - reusedPFServed []float64 latencyMetrics []*metrics.Metric - replayedNum int // only valid for lazy serving uniqueNum int currentMetric *metrics.Metric } @@ -99,11 +178,8 @@ func NewSnapshotState(cfg SnapshotStateCfg) *SnapshotState { s := new(SnapshotState) s.SnapshotStateCfg = cfg - s.trace = initTrace(s.getTraceFile()) if s.metricsModeOn { - s.totalPFServed = make([]float64, 0) s.uniquePFServed = make([]float64, 0) - s.reusedPFServed = make([]float64, 0) s.latencyMetrics = make([]*metrics.Metric, 0) } @@ -113,18 +189,16 @@ func NewSnapshotState(cfg SnapshotStateCfg) *SnapshotState { func (s *SnapshotState) setupStateOnActivate() { s.isActive = true s.isEverActivated = true - s.firstPageFaultOnce = new(sync.Once) s.quitCh = make(chan int, 1) if s.metricsModeOn { s.uniqueNum = 0 - s.replayedNum = 0 s.currentMetric = metrics.NewMetric() } } -func (s *SnapshotState) getUFFD() error { - mappings, userFaultFD, err := receiveUffdMappingsAndFDFromSocket(s.InstanceSockAddr) +func (s *SnapshotState) getUFFD(socketReadyCh chan<- error) error { + mappings, userFaultFD, err := receiveUffdMappingsAndFDFromSocket(s.InstanceSockAddr, socketReadyCh) if err != nil { log.Error("Failed to receive the uffd and guest memory mappings") return err @@ -136,118 +210,175 @@ func (s *SnapshotState) getUFFD() error { return nil } -func (s *SnapshotState) processMetrics() { - if s.metricsModeOn && s.isRecordReady { - s.uniquePFServed = append(s.uniquePFServed, float64(s.uniqueNum)) +func receiveUffdMappingsAndFDFromSocket(socketPath string, socketReadyCh chan<- error) ([]GuestRegionUffdMapping, *os.File, error) { + if socketPath == "" { + notifySocketReady(socketReadyCh, errEmptyUffdSocketPath) + return nil, nil, errEmptyUffdSocketPath + } + if err := os.MkdirAll(filepath.Dir(socketPath), 0755); err != nil { + notifySocketReady(socketReadyCh, err) + return nil, nil, err + } + if err := removeStaleUffdSocket(socketPath); err != nil { + notifySocketReady(socketReadyCh, err) + return nil, nil, err + } - if s.IsLazyMode { - s.totalPFServed = append(s.totalPFServed, float64(s.replayedNum)) - s.reusedPFServed = append( - s.reusedPFServed, - float64(s.replayedNum-s.uniqueNum), - ) - } + addr := &net.UnixAddr{Name: socketPath, Net: "unix"} + listener, err := net.ListenUnix("unix", addr) + if err != nil { + notifySocketReady(socketReadyCh, err) + return nil, nil, err + } + defer func() { _ = listener.Close() }() + defer func() { _ = os.Remove(socketPath) }() - s.latencyMetrics = append(s.latencyMetrics, s.currentMetric) + if err := listener.SetDeadline(time.Now().Add(uffdSocketAcceptTimeout)); err != nil { + notifySocketReady(socketReadyCh, err) + return nil, nil, err + } + + notifySocketReady(socketReadyCh, nil) + + conn, err := listener.AcceptUnix() + if err != nil { + return nil, nil, err } + defer func() { _ = conn.Close() }() + + return receiveUffdMappingsAndFD(conn) } -func (s *SnapshotState) getTraceFile() string { - return filepath.Join(s.BaseDir, "trace") +func notifySocketReady(socketReadyCh chan<- error, err error) { + if socketReadyCh == nil { + return + } + socketReadyCh <- err } -func (s *SnapshotState) mapGuestMemory() error { - fd, err := os.OpenFile(s.GuestMemPath, os.O_RDONLY, 0444) - if err != nil { - log.Errorf("Failed to open guest memory file: %v", err) - return err +func receiveUffdMappingsAndFD(conn *net.UnixConn) ([]GuestRegionUffdMapping, *os.File, error) { + if err := conn.SetReadDeadline(time.Now().Add(uffdSocketReadTimeout)); err != nil { + return nil, nil, err } - defer func() { _ = fd.Close() }() + defer func() { _ = conn.SetReadDeadline(time.Time{}) }() - s.guestMem, err = unix.Mmap(int(fd.Fd()), 0, s.GuestMemSize, unix.PROT_READ, unix.MAP_PRIVATE) + body := make([]byte, uffdSocketPayloadSize) + oob := make([]byte, unix.CmsgSpace(uffdSocketFDLimit*4)) + + n, oobn, flags, _, err := conn.ReadMsgUnix(body, oob) if err != nil { - log.Errorf("Failed to mmap guest memory file: %v", err) - return err + return nil, nil, err + } + if flags&unix.MSG_TRUNC != 0 { + return nil, nil, errors.New("uffd mappings payload was truncated") + } + if flags&unix.MSG_CTRUNC != 0 { + return nil, nil, errors.New("uffd fd control message was truncated") } - return nil -} + fds, err := parseUnixRights(oob[:oobn]) + if err != nil { + return nil, nil, err + } + if len(fds) != 1 { + closeFDs(fds) + return nil, nil, fmt.Errorf("%w: got %d", errUnexpectedUffdFDCount, len(fds)) + } -func (s *SnapshotState) unmapGuestMemory() error { - if err := unix.Munmap(s.guestMem); err != nil { - log.Errorf("Failed to munmap guest memory file: %v", err) - return err + uffdFile := os.NewFile(uintptr(fds[0]), "userfaultfd") + if uffdFile == nil { + return nil, nil, errors.New("failed to create file for uffd fd") } - return nil -} + var mappings []GuestRegionUffdMapping + if err := json.Unmarshal(body[:n], &mappings); err != nil { + _ = uffdFile.Close() + return nil, nil, fmt.Errorf("cannot deserialize memory mappings: %w", err) + } + if len(mappings) == 0 { + _ = uffdFile.Close() + return nil, nil, errNoGuestRegionMappings + } -// alignment returns alignment of the block in memory -// with reference to alignSize -// -// Can't check alignment of a zero sized block as &block[0] is invalid -func alignment(block []byte, alignSize int) int { - return int(uintptr(unsafe.Pointer(&block[0])) & uintptr(alignSize-1)) + return mappings, uffdFile, nil } -// AlignedBlock returns []byte of size BlockSize aligned to a multiple -// of alignSize in memory (must be power of two) -func AlignedBlock(blockSize int) []byte { - alignSize := os.Getpagesize() // must be multiple of the filesystem block size +func parseUnixRights(oob []byte) ([]int, error) { + scms, err := unix.ParseSocketControlMessage(oob) + if err != nil { + return nil, err + } - if blockSize == 0 { - return nil + var fds []int + for i := range scms { + rights, err := unix.ParseUnixRights(&scms[i]) + if err != nil { + closeFDs(fds) + return nil, err + } + fds = append(fds, rights...) } - block := make([]byte, blockSize+alignSize) + return fds, nil +} - a := alignment(block, alignSize) - offset := 0 - if a != 0 { - offset = alignSize - a +func closeFDs(fds []int) { + for _, receivedFD := range fds { + _ = unix.Close(receivedFD) } - block = block[offset : offset+blockSize] +} - // Check - if blockSize != 0 { - a = alignment(block, alignSize) - if a != 0 { - log.Fatal("Failed to align block") +func removeStaleUffdSocket(socketPath string) error { + info, err := os.Lstat(socketPath) + if err == nil { + if info.Mode()&os.ModeSocket == 0 { + return fmt.Errorf("refusing to remove non-socket uffd path %q", socketPath) } + return os.Remove(socketPath) + } + if os.IsNotExist(err) { + return nil } - return block + return err } -// fetchState Fetches the working set file (or the whole guest memory) and the VMM state file -func (s *SnapshotState) fetchState() error { - if _, err := os.ReadFile(s.VMMStatePath); err != nil { - log.Errorf("Failed to fetch VMM state: %v\n", err) - return err +func (s *SnapshotState) processMetrics() { + if s.metricsModeOn && s.currentMetric != nil { + s.uniquePFServed = append(s.uniquePFServed, float64(s.uniqueNum)) + s.latencyMetrics = append(s.latencyMetrics, s.currentMetric) } +} - if !s.IsLazyMode { - return nil +func (s *SnapshotState) mapGuestMemory() error { + fd, err := os.OpenFile(s.GuestMemPath, os.O_RDONLY, 0444) + if err != nil { + log.Errorf("Failed to open guest memory file: %v", err) + return err } + defer func() { _ = fd.Close() }() - size := len(s.trace.trace) * os.Getpagesize() - - // O_DIRECT allows to fully leverage disk bandwidth by bypassing the OS page cache - f, err := os.OpenFile(s.WorkingSetPath, os.O_RDONLY|syscall.O_DIRECT, 0600) + s.guestMem, err = unix.Mmap(int(fd.Fd()), 0, s.GuestMemSize, unix.PROT_READ, unix.MAP_PRIVATE) if err != nil { - log.Errorf("Failed to open the working set file for direct-io: %v\n", err) + log.Errorf("Failed to mmap guest memory file: %v", err) return err } - s.workingSet = AlignedBlock(size) // direct io requires aligned buffer + return nil +} - if n, err := f.Read(s.workingSet); n != size || err != nil { - log.Errorf("Reading working set file failed: %v\n", err) +func (s *SnapshotState) unmapGuestMemory() error { + if err := unix.Munmap(s.guestMem); err != nil { + log.Errorf("Failed to munmap guest memory file: %v", err) return err } - log.Debug("Fetched the entire working set") - if err := f.Close(); err != nil { - log.Errorf("Failed to close the working set file: %v\n", err) + return nil +} + +// fetchState verifies the VMM state file before snapshot activation. +func (s *SnapshotState) fetchState() error { + if _, err := os.ReadFile(s.VMMStatePath); err != nil { + log.Errorf("Failed to fetch VMM state: %v\n", err) return err } @@ -381,31 +512,7 @@ func (s *SnapshotState) registerEpoller() error { } func (s *SnapshotState) servePageFault(fd int, address uint64) error { - var ( - tStart time.Time - workingSetInstalled bool - ) - - s.firstPageFaultOnce.Do( - func() { - s.startAddress = address - - if s.isRecordReady && !s.IsLazyMode { - if s.metricsModeOn { - tStart = time.Now() - } - s.installWorkingSetPages(fd) - if s.metricsModeOn { - s.currentMetric.MetricMap[installWSMetric] = metrics.ToUS(time.Since(tStart)) - } - - workingSetInstalled = true - } - }) - - if workingSetInstalled { - return nil - } + var tStart time.Time copyArgs, err := pageFaultCopyArgsForFault(s.guestRegionMappings, address) if err != nil { @@ -417,29 +524,8 @@ func (s *SnapshotState) servePageFault(fd int, address uint64) error { return err } - rec := Record{ - offset: copyArgs.srcOffset, - } - - if !s.isRecordReady { - s.trace.AppendRecord(rec) - } else { - log.Debug("Serving a page that is missing from the working set") - } - if s.metricsModeOn { - if s.isRecordReady { - if s.IsLazyMode { - if !s.trace.containsRecord(rec) { - s.uniqueNum++ - } - s.replayedNum++ - } else { - s.uniqueNum++ - } - - } - + s.uniqueNum++ tStart = time.Now() } @@ -452,41 +538,6 @@ func (s *SnapshotState) servePageFault(fd int, address uint64) error { return err } -func (s *SnapshotState) installWorkingSetPages(fd int) { - log.Debug("Installing the working set pages") - - // build a list of sorted regions - keys := make([]uint64, 0) - for k := range s.trace.regions { - keys = append(keys, k) - } - sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) - - var ( - srcOffset uint64 - ) - - for _, offset := range keys { - regLength := s.trace.regions[offset] - regAddress := s.startAddress + offset - mode := uint64(C.const_UFFDIO_COPY_MODE_DONTWAKE) - src := uint64(uintptr(unsafe.Pointer(&s.workingSet[srcOffset]))) - dst := regAddress - - if err := installRegion(fd, src, dst, mode, uint64(regLength)); err != nil { - log.Fatalf("install_region: %v", err) - } - - srcOffset += uint64(regLength) * 4096 - } - - wake(fd, s.startAddress, os.Getpagesize()) -} - -func installRegion(fd int, src, dst, mode, pageCount uint64) error { - return installRegionBytes(fd, src, dst, mode, uint64(os.Getpagesize())*pageCount) -} - func installRegionBytes(fd int, src, dst, mode, length uint64) error { cUC := C.struct_uffdio_copy{ mode: C.ulonglong(mode), @@ -535,23 +586,6 @@ func ioctl(fd uintptr, request int, argp unsafe.Pointer) error { return nil } -func wake(fd int, startAddress uint64, len int) { - cUR := C.struct_uffdio_range{ - start: C.ulonglong(startAddress), - len: C.ulonglong(len), - } - - err := ioctl(uintptr(fd), int(C.const_UFFDIO_WAKE), unsafe.Pointer(&cUR)) - if err != nil { - log.Fatalf("ioctl failed: %v", err) - } -} - -//nolint:unused -func registerForUpf(startAddress []byte, len uint64) int { - return int(C.register_for_upf(unsafe.Pointer(&startAddress[0]), C.ulong(len))) -} - func sizeOfUFFDMsg() int { return C.sizeof_struct_uffd_msg } diff --git a/memory/manager/uffd_socket_test.go b/memory/manager/snapshot_state_test.go similarity index 65% rename from memory/manager/uffd_socket_test.go rename to memory/manager/snapshot_state_test.go index de5c8f0d2..4a311d289 100644 --- a/memory/manager/uffd_socket_test.go +++ b/memory/manager/snapshot_state_test.go @@ -15,6 +15,156 @@ import ( "golang.org/x/sys/unix" ) +func TestPageAlignFaultAddress(t *testing.T) { + region := GuestRegionUffdMapping{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + PageSize: 0x1000, + } + + got, err := pageAlignFaultAddress(0x101234, region) + if err != nil { + t.Fatalf("pageAlignFaultAddress returned error: %v", err) + } + if want := uint64(0x101000); got != want { + t.Fatalf("pageAlignFaultAddress() = %#x, want %#x", got, want) + } +} + +func TestPageFaultCopyArgsForFault(t *testing.T) { + tests := []struct { + name string + regions []GuestRegionUffdMapping + fault uint64 + want pageFaultCopyArgs + }{ + { + name: "zero offset", + regions: []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + PageSize: 0x1000, + }}, + fault: 0x102000, + want: pageFaultCopyArgs{ + srcOffset: 0x2000, + dstAddr: 0x102000, + copyLen: 0x1000, + copyMode: 0, + }, + }, + { + name: "non-zero offset", + regions: []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + Offset: 0x800000, + PageSize: 0x1000, + }}, + fault: 0x103000, + want: pageFaultCopyArgs{ + srcOffset: 0x803000, + dstAddr: 0x103000, + copyLen: 0x1000, + copyMode: 0, + }, + }, + { + name: "multiple regions", + regions: []GuestRegionUffdMapping{ + { + BaseHostVirtAddr: 0x100000, + Size: 0x2000, + PageSize: 0x1000, + }, + { + BaseHostVirtAddr: 0x200000, + Size: 0x3000, + Offset: 0x900000, + PageSize: 0x1000, + }, + }, + fault: 0x201000, + want: pageFaultCopyArgs{ + srcOffset: 0x901000, + dstAddr: 0x201000, + copyLen: 0x1000, + copyMode: 0, + }, + }, + { + name: "not page-aligned", + regions: []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x4000, + Offset: 0x300000, + PageSize: 0x1000, + }}, + fault: 0x101234, + want: pageFaultCopyArgs{ + srcOffset: 0x301000, + dstAddr: 0x101000, + copyLen: 0x1000, + copyMode: 0, + }, + }, + { + name: "larger page size", + regions: []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x8000, + Offset: 0x500000, + PageSize: 0x2000, + }}, + fault: 0x103456, + want: pageFaultCopyArgs{ + srcOffset: 0x502000, + dstAddr: 0x102000, + copyLen: 0x2000, + copyMode: 0, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := pageFaultCopyArgsForFault(tt.regions, tt.fault) + if err != nil { + t.Fatalf("pageFaultCopyArgsForFault returned error: %v", err) + } + if got != tt.want { + t.Fatalf("pageFaultCopyArgsForFault() = %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestPageFaultCopyArgsForFaultOutsideAllRegions(t *testing.T) { + regions := []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x2000, + PageSize: 0x1000, + }} + + _, err := pageFaultCopyArgsForFault(regions, 0x103000) + if !errors.Is(err, errGuestRegionNotFound) { + t.Fatalf("pageFaultCopyArgsForFault() error = %v, want %v", err, errGuestRegionNotFound) + } +} + +func TestPageFaultCopyArgsForFaultZeroPageSize(t *testing.T) { + regions := []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x2000, + PageSize: 0, + }} + + _, err := pageFaultCopyArgsForFault(regions, 0x100000) + if !errors.Is(err, errInvalidGuestRegionPageSize) { + t.Fatalf("pageFaultCopyArgsForFault() error = %v, want %v", err, errInvalidGuestRegionPageSize) + } +} + func TestReceiveUffdMappingsAndFD(t *testing.T) { mappings := []GuestRegionUffdMapping{{ BaseHostVirtAddr: 0x100000, @@ -125,7 +275,7 @@ func TestSnapshotStateGetUFFDStoresMappingsAndFD(t *testing.T) { stateErrCh := make(chan error, 1) go func() { - stateErrCh <- state.getUFFD() + stateErrCh <- state.getUFFD(nil) }() conn := dialUnixSocketWithRetry(t, socketPath) diff --git a/memory/manager/trace.go b/memory/manager/trace.go deleted file mode 100644 index e5b9f3242..000000000 --- a/memory/manager/trace.go +++ /dev/null @@ -1,210 +0,0 @@ -// MIT License -// -// Copyright (c) 2020 Dmitrii Ustiugov, Plamen Petrov and EASE lab -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package manager - -import ( - "encoding/csv" - "os" - "sort" - "strconv" - "sync" - - log "github.com/sirupsen/logrus" -) - -// Record A tuple with an address -type Record struct { - offset uint64 -} - -// Trace Contains records -type Trace struct { - sync.Mutex - traceFileName string - - containedOffsets map[uint64]int - trace []Record - regions map[uint64]int -} - -func initTrace(traceFileName string) *Trace { - t := new(Trace) - - t.traceFileName = traceFileName - t.regions = make(map[uint64]int) - t.containedOffsets = make(map[uint64]int) - t.trace = make([]Record, 0) - - return t -} - -// AppendRecord Appends a record to the trace -func (t *Trace) AppendRecord(r Record) { - t.Lock() - defer t.Unlock() - - t.trace = append(t.trace, r) - t.containedOffsets[r.offset] = 0 -} - -// WriteTrace Writes all the records to a file -func (t *Trace) WriteTrace() { - t.Lock() - defer t.Unlock() - - file, err := os.Create(t.traceFileName) - if err != nil { - log.Fatalf("Failed to open trace file for writing: %v", err) - } - defer func() { _ = file.Close() }() - - writer := csv.NewWriter(file) - defer writer.Flush() - - for _, rec := range t.trace { - err := writer.Write([]string{ - strconv.FormatUint(rec.offset, 16)}) - if err != nil { - log.Fatalf("Failed to write trace: %v", err) - } - } -} - -// readTrace Reads all the records from a CSV file -// -//nolint:unused -func (t *Trace) readTrace() { - f, err := os.Open(t.traceFileName) - if err != nil { - log.Fatalf("Failed to open trace file for reading: %v", err) - } - defer func() { _ = f.Close() }() - - lines, err := csv.NewReader(f).ReadAll() - if err != nil { - log.Fatalf("Failed to read from the trace file: %v", err) - } - - for _, line := range lines { - rec := readRecord(line) - t.AppendRecord(rec) - } -} - -// readRecord Parses a record from a line -// -//nolint:unused -func readRecord(line []string) Record { - offset, err := strconv.ParseUint(line[0], 16, 64) - if err != nil { - log.Fatalf("Failed to convert string to offset: %v", err) - } - - rec := Record{ - offset: offset, - } - return rec -} - -// Search trace for the record with the same offset -func (t *Trace) containsRecord(rec Record) bool { - _, ok := t.containedOffsets[rec.offset] - - return ok -} - -// ProcessRecord Prepares the trace, the regions map, and the working set file for replay -// Must be called when record is done (i.e., it is not concurrency-safe vs. AppendRecord) -func (t *Trace) ProcessRecord(GuestMemPath, WorkingSetPath string) { - log.Debug("Preparing replay structures") - - // sort trace records in the ascending order by offset - sort.Slice(t.trace, func(i, j int) bool { - return t.trace[i].offset < t.trace[j].offset - }) - - // build the map of contiguous regions from the trace records - var last, regionStart uint64 - for _, rec := range t.trace { - if rec.offset != last+uint64(os.Getpagesize()) { - regionStart = rec.offset - t.regions[regionStart] = 1 - } else { - t.regions[regionStart]++ - } - - last = rec.offset - } - - t.writeWorkingSetPagesToFile(GuestMemPath, WorkingSetPath) -} - -func (t *Trace) writeWorkingSetPagesToFile(guestMemFileName, WorkingSetPath string) { - log.Debug("Writing the working set pages to a disk") - - fSrc, err := os.Open(guestMemFileName) - if err != nil { - log.Fatalf("Failed to open guest memory file for reading") - } - defer func() { _ = fSrc.Close() }() - fDst, err := os.Create(WorkingSetPath) - if err != nil { - log.Fatalf("Failed to open ws file for writing") - } - defer func() { _ = fDst.Close() }() - - var ( - dstOffset int64 - count int - ) - - // Form a sorted slice of keys to access the map in a predetermined order - keys := make([]uint64, 0) - for k := range t.regions { - keys = append(keys, k) - } - sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) - - for _, offset := range keys { - regLength := t.regions[offset] - copyLen := regLength * os.Getpagesize() - - buf := make([]byte, copyLen) - - if n, err := fSrc.ReadAt(buf, int64(offset)); n != copyLen || err != nil { - log.Fatalf("Read file failed for src") - } - - if n, err := fDst.WriteAt(buf, dstOffset); n != copyLen || err != nil { - log.Fatalf("Write file failed for dst") - } - - dstOffset += int64(copyLen) - - count += regLength - } - - if err := fDst.Sync(); err != nil { - log.Fatalf("Sync file failed for dst") - } -} diff --git a/memory/manager/uffd_mapping.go b/memory/manager/uffd_mapping.go deleted file mode 100644 index 0458fb0d9..000000000 --- a/memory/manager/uffd_mapping.go +++ /dev/null @@ -1,104 +0,0 @@ -package manager - -import ( - "errors" - "fmt" - "math" -) - -var ( - errInvalidGuestRegionPageSize = errors.New("guest region page size must be non-zero") - errGuestRegionNotFound = errors.New("fault address is outside guest memory mappings") -) - -// GuestRegionUffdMapping describes Firecracker's UFFD guest memory mapping. -type GuestRegionUffdMapping struct { - BaseHostVirtAddr uint64 `json:"base_host_virt_addr"` - Size uint64 `json:"size"` - Offset uint64 `json:"offset"` - PageSize uint64 `json:"page_size"` -} - -type pageFaultCopyArgs struct { - srcOffset uint64 - dstAddr uint64 - copyLen uint64 - copyMode uint64 -} - -func pageAlignFaultAddress(faultAddr uint64, region GuestRegionUffdMapping) (uint64, error) { - if region.PageSize == 0 { - return 0, errInvalidGuestRegionPageSize - } - - return faultAddr - faultAddr%region.PageSize, nil -} - -func findGuestRegionForFaultPage(regions []GuestRegionUffdMapping, faultPageAddr uint64) (GuestRegionUffdMapping, error) { - for _, region := range regions { - if regionContainsFaultPage(region, faultPageAddr) { - return region, nil - } - } - - return GuestRegionUffdMapping{}, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultPageAddr) -} - -func guestMemoryOffsetForFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) (uint64, error) { - if region.PageSize == 0 { - return 0, errInvalidGuestRegionPageSize - } - if !regionContainsFaultPage(region, faultPageAddr) { - return 0, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultPageAddr) - } - - regionOffset := faultPageAddr - region.BaseHostVirtAddr - if region.Offset > math.MaxUint64-regionOffset { - return 0, fmt.Errorf("guest memory offset overflow for fault address %#x", faultPageAddr) - } - - return region.Offset + regionOffset, nil -} - -func guestMemoryOffsetForFault(regions []GuestRegionUffdMapping, faultAddr uint64) (uint64, error) { - copyArgs, err := pageFaultCopyArgsForFault(regions, faultAddr) - if err != nil { - return 0, err - } - - return copyArgs.srcOffset, nil -} - -func pageFaultCopyArgsForFault(regions []GuestRegionUffdMapping, faultAddr uint64) (pageFaultCopyArgs, error) { - for _, region := range regions { - faultPageAddr, err := pageAlignFaultAddress(faultAddr, region) - if err != nil { - return pageFaultCopyArgs{}, err - } - if !regionContainsFaultPage(region, faultPageAddr) { - continue - } - - srcOffset, err := guestMemoryOffsetForFaultPage(region, faultPageAddr) - if err != nil { - return pageFaultCopyArgs{}, err - } - - return pageFaultCopyArgs{ - srcOffset: srcOffset, - dstAddr: faultPageAddr, - copyLen: region.PageSize, - copyMode: 0, - }, nil - } - - return pageFaultCopyArgs{}, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultAddr) -} - -func regionContainsFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) bool { - if region.Size == 0 || faultPageAddr < region.BaseHostVirtAddr { - return false - } - - return faultPageAddr-region.BaseHostVirtAddr < region.Size -} diff --git a/memory/manager/uffd_mapping_test.go b/memory/manager/uffd_mapping_test.go deleted file mode 100644 index 2b1203a95..000000000 --- a/memory/manager/uffd_mapping_test.go +++ /dev/null @@ -1,242 +0,0 @@ -package manager - -import ( - "errors" - "testing" -) - -func TestPageAlignFaultAddress(t *testing.T) { - region := GuestRegionUffdMapping{ - BaseHostVirtAddr: 0x100000, - Size: 0x4000, - PageSize: 0x1000, - } - - got, err := pageAlignFaultAddress(0x101234, region) - if err != nil { - t.Fatalf("pageAlignFaultAddress returned error: %v", err) - } - if want := uint64(0x101000); got != want { - t.Fatalf("pageAlignFaultAddress() = %#x, want %#x", got, want) - } -} - -func TestFindGuestRegionForFaultPage(t *testing.T) { - regions := []GuestRegionUffdMapping{ - { - BaseHostVirtAddr: 0x100000, - Size: 0x2000, - PageSize: 0x1000, - }, - { - BaseHostVirtAddr: 0x200000, - Size: 0x3000, - Offset: 0x8000, - PageSize: 0x1000, - }, - } - - got, err := findGuestRegionForFaultPage(regions, 0x201000) - if err != nil { - t.Fatalf("findGuestRegionForFaultPage returned error: %v", err) - } - if want := regions[1]; got != want { - t.Fatalf("findGuestRegionForFaultPage() = %+v, want %+v", got, want) - } -} - -func TestGuestMemoryOffsetForFault(t *testing.T) { - tests := []struct { - name string - regions []GuestRegionUffdMapping - fault uint64 - want uint64 - }{ - { - name: "one region with zero offset", - regions: []GuestRegionUffdMapping{{ - BaseHostVirtAddr: 0x100000, - Size: 0x4000, - PageSize: 0x1000, - }}, - fault: 0x102000, - want: 0x2000, - }, - { - name: "one region with non-zero offset", - regions: []GuestRegionUffdMapping{{ - BaseHostVirtAddr: 0x100000, - Size: 0x4000, - Offset: 0x800000, - PageSize: 0x1000, - }}, - fault: 0x103000, - want: 0x803000, - }, - { - name: "multiple regions", - regions: []GuestRegionUffdMapping{ - { - BaseHostVirtAddr: 0x100000, - Size: 0x2000, - PageSize: 0x1000, - }, - { - BaseHostVirtAddr: 0x200000, - Size: 0x3000, - Offset: 0x900000, - PageSize: 0x1000, - }, - }, - fault: 0x201000, - want: 0x901000, - }, - { - name: "address not page-aligned", - regions: []GuestRegionUffdMapping{{ - BaseHostVirtAddr: 0x100000, - Size: 0x4000, - Offset: 0x300000, - PageSize: 0x1000, - }}, - fault: 0x101234, - want: 0x301000, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := guestMemoryOffsetForFault(tt.regions, tt.fault) - if err != nil { - t.Fatalf("guestMemoryOffsetForFault returned error: %v", err) - } - if got != tt.want { - t.Fatalf("guestMemoryOffsetForFault() = %#x, want %#x", got, tt.want) - } - }) - } -} - -func TestGuestMemoryOffsetForFaultOutsideAllRegions(t *testing.T) { - regions := []GuestRegionUffdMapping{{ - BaseHostVirtAddr: 0x100000, - Size: 0x2000, - PageSize: 0x1000, - }} - - _, err := guestMemoryOffsetForFault(regions, 0x103000) - if !errors.Is(err, errGuestRegionNotFound) { - t.Fatalf("guestMemoryOffsetForFault() error = %v, want %v", err, errGuestRegionNotFound) - } -} - -func TestGuestMemoryOffsetForFaultZeroPageSize(t *testing.T) { - regions := []GuestRegionUffdMapping{{ - BaseHostVirtAddr: 0x100000, - Size: 0x2000, - PageSize: 0, - }} - - _, err := guestMemoryOffsetForFault(regions, 0x100000) - if !errors.Is(err, errInvalidGuestRegionPageSize) { - t.Fatalf("guestMemoryOffsetForFault() error = %v, want %v", err, errInvalidGuestRegionPageSize) - } -} - -func TestPageFaultCopyArgsForFault(t *testing.T) { - tests := []struct { - name string - regions []GuestRegionUffdMapping - fault uint64 - want pageFaultCopyArgs - }{ - { - name: "zero offset", - regions: []GuestRegionUffdMapping{{ - BaseHostVirtAddr: 0x100000, - Size: 0x4000, - PageSize: 0x1000, - }}, - fault: 0x102000, - want: pageFaultCopyArgs{ - srcOffset: 0x2000, - dstAddr: 0x102000, - copyLen: 0x1000, - copyMode: 0, - }, - }, - { - name: "non-zero offset", - regions: []GuestRegionUffdMapping{{ - BaseHostVirtAddr: 0x100000, - Size: 0x4000, - Offset: 0x800000, - PageSize: 0x1000, - }}, - fault: 0x103000, - want: pageFaultCopyArgs{ - srcOffset: 0x803000, - dstAddr: 0x103000, - copyLen: 0x1000, - copyMode: 0, - }, - }, - { - name: "not page-aligned", - regions: []GuestRegionUffdMapping{{ - BaseHostVirtAddr: 0x100000, - Size: 0x4000, - Offset: 0x300000, - PageSize: 0x1000, - }}, - fault: 0x101234, - want: pageFaultCopyArgs{ - srcOffset: 0x301000, - dstAddr: 0x101000, - copyLen: 0x1000, - copyMode: 0, - }, - }, - { - name: "larger page size", - regions: []GuestRegionUffdMapping{{ - BaseHostVirtAddr: 0x100000, - Size: 0x8000, - Offset: 0x500000, - PageSize: 0x2000, - }}, - fault: 0x103456, - want: pageFaultCopyArgs{ - srcOffset: 0x502000, - dstAddr: 0x102000, - copyLen: 0x2000, - copyMode: 0, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := pageFaultCopyArgsForFault(tt.regions, tt.fault) - if err != nil { - t.Fatalf("pageFaultCopyArgsForFault returned error: %v", err) - } - if got != tt.want { - t.Fatalf("pageFaultCopyArgsForFault() = %+v, want %+v", got, tt.want) - } - }) - } -} - -func TestPageFaultCopyArgsForFaultOutsideAllRegions(t *testing.T) { - regions := []GuestRegionUffdMapping{{ - BaseHostVirtAddr: 0x100000, - Size: 0x2000, - PageSize: 0x1000, - }} - - _, err := pageFaultCopyArgsForFault(regions, 0x103000) - if !errors.Is(err, errGuestRegionNotFound) { - t.Fatalf("pageFaultCopyArgsForFault() error = %v, want %v", err, errGuestRegionNotFound) - } -} diff --git a/memory/manager/uffd_socket.go b/memory/manager/uffd_socket.go deleted file mode 100644 index 728f10f4f..000000000 --- a/memory/manager/uffd_socket.go +++ /dev/null @@ -1,144 +0,0 @@ -package manager - -import ( - "encoding/json" - "errors" - "fmt" - "net" - "os" - "path/filepath" - "time" - - "golang.org/x/sys/unix" -) - -const ( - uffdSocketPayloadSize = 64 * 1024 - uffdSocketFDLimit = 2 - uffdSocketReadTimeout = time.Second - uffdSocketAcceptTimeout = 30 * time.Second -) - -var ( - errUnexpectedUffdFDCount = errors.New("expected exactly one uffd fd") - errNoGuestRegionMappings = errors.New("no guest region mappings received") - errEmptyUffdSocketPath = errors.New("empty uffd socket path") -) - -func receiveUffdMappingsAndFDFromSocket(socketPath string) ([]GuestRegionUffdMapping, *os.File, error) { - if socketPath == "" { - return nil, nil, errEmptyUffdSocketPath - } - if err := os.MkdirAll(filepath.Dir(socketPath), 0755); err != nil { - return nil, nil, err - } - if err := removeStaleUffdSocket(socketPath); err != nil { - return nil, nil, err - } - - addr := &net.UnixAddr{Name: socketPath, Net: "unix"} - listener, err := net.ListenUnix("unix", addr) - if err != nil { - return nil, nil, err - } - defer func() { _ = listener.Close() }() - defer func() { _ = os.Remove(socketPath) }() - - if err := listener.SetDeadline(time.Now().Add(uffdSocketAcceptTimeout)); err != nil { - return nil, nil, err - } - - conn, err := listener.AcceptUnix() - if err != nil { - return nil, nil, err - } - defer func() { _ = conn.Close() }() - - return receiveUffdMappingsAndFD(conn) -} - -func receiveUffdMappingsAndFD(conn *net.UnixConn) ([]GuestRegionUffdMapping, *os.File, error) { - if err := conn.SetReadDeadline(time.Now().Add(uffdSocketReadTimeout)); err != nil { - return nil, nil, err - } - defer func() { _ = conn.SetReadDeadline(time.Time{}) }() - - body := make([]byte, uffdSocketPayloadSize) - oob := make([]byte, unix.CmsgSpace(uffdSocketFDLimit*4)) - - n, oobn, flags, _, err := conn.ReadMsgUnix(body, oob) - if err != nil { - return nil, nil, err - } - if flags&unix.MSG_TRUNC != 0 { - return nil, nil, errors.New("uffd mappings payload was truncated") - } - if flags&unix.MSG_CTRUNC != 0 { - return nil, nil, errors.New("uffd fd control message was truncated") - } - - fds, err := parseUnixRights(oob[:oobn]) - if err != nil { - return nil, nil, err - } - if len(fds) != 1 { - closeFDs(fds) - return nil, nil, fmt.Errorf("%w: got %d", errUnexpectedUffdFDCount, len(fds)) - } - - uffdFile := os.NewFile(uintptr(fds[0]), "userfaultfd") - if uffdFile == nil { - return nil, nil, errors.New("failed to create file for uffd fd") - } - - var mappings []GuestRegionUffdMapping - if err := json.Unmarshal(body[:n], &mappings); err != nil { - _ = uffdFile.Close() - return nil, nil, fmt.Errorf("cannot deserialize memory mappings: %w", err) - } - if len(mappings) == 0 { - _ = uffdFile.Close() - return nil, nil, errNoGuestRegionMappings - } - - return mappings, uffdFile, nil -} - -func parseUnixRights(oob []byte) ([]int, error) { - scms, err := unix.ParseSocketControlMessage(oob) - if err != nil { - return nil, err - } - - var fds []int - for i := range scms { - rights, err := unix.ParseUnixRights(&scms[i]) - if err != nil { - closeFDs(fds) - return nil, err - } - fds = append(fds, rights...) - } - - return fds, nil -} - -func closeFDs(fds []int) { - for _, receivedFD := range fds { - _ = unix.Close(receivedFD) - } -} - -func removeStaleUffdSocket(socketPath string) error { - info, err := os.Lstat(socketPath) - if err == nil { - if info.Mode()&os.ModeSocket == 0 { - return fmt.Errorf("refusing to remove non-socket uffd path %q", socketPath) - } - return os.Remove(socketPath) - } - if os.IsNotExist(err) { - return nil - } - return err -} diff --git a/memory/manager/user_page_faults.h b/memory/manager/user_page_faults.h index aa45065c7..a9f80b056 100644 --- a/memory/manager/user_page_faults.h +++ b/memory/manager/user_page_faults.h @@ -22,44 +22,9 @@ #define _GNU_SOURCE -#include #include #include -#include -#include -#include -#include -#include -#include // constants for use from Go -int const_UFFDIO_WAKE = UFFDIO_WAKE; int const_UFFDIO_COPY = UFFDIO_COPY; int const_UFFD_EVENT_PAGEFAULT = UFFD_EVENT_PAGEFAULT; -int const_UFFDIO_COPY_MODE_DONTWAKE = UFFDIO_COPY_MODE_DONTWAKE; - -#define errExit(msg) \ - do { perror(msg); exit(EXIT_FAILURE); } while (0) - -long register_for_upf(void *start_address, unsigned long len) { - struct uffdio_api uffdio_api; - struct uffdio_register uffdio_register; - long uffd; - - uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); - if (uffd == -1) - errExit("userfaultfd"); - - uffdio_api.api = UFFD_API; - uffdio_api.features = 0; - if (ioctl(uffd, UFFDIO_API, &uffdio_api) == -1) - errExit("ioctl-UFFDIO_API"); - - uffdio_register.range.start = (unsigned long) start_address; - uffdio_register.range.len = len; - uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING; - if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) - errExit("ioctl-UFFDIO_REGISTER"); - - return uffd; -} From a49d0fe54e4e6b03b88cc7180844c5bad7ae2fd3 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Mon, 29 Jun 2026 10:41:00 +0800 Subject: [PATCH 09/21] Run memory manager tests in CI --- .github/workflows/unit_tests.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 6fb425372..d446879ab 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -20,6 +20,24 @@ env: GO111MODULE: on jobs: + memory-manager-unit-test: + name: "Unit tests: memory manager" + runs-on: ubuntu-24.04 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v6 + + - name: Set up Go version in go.mod file + uses: actions/setup-go@v6 + with: + go-version-file: ${{ github.workspace }}/go.mod + cache-dependency-path: | + **/go.sum + **/go.mod + + - name: Run memory manager tests + run: go test ./memory/manager + unit-test: name: Unit test runs-on: ubuntu-24.04 From 0a2c3db40403d28c5ebfd50b99fb102906d93616 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Mon, 29 Jun 2026 11:07:16 +0800 Subject: [PATCH 10/21] Fix UFFD poller shutdown race --- .github/workflows/unit_tests.yml | 4 +- memory/manager/manager.go | 8 ++-- memory/manager/manager_test.go | 13 ------ memory/manager/snapshot_state.go | 71 +++++++++++++++++++++++++++++++- 4 files changed, 77 insertions(+), 19 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index d446879ab..90a7097e5 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -36,7 +36,9 @@ jobs: **/go.mod - name: Run memory manager tests - run: go test ./memory/manager + run: | + go test ./memory/manager + go test -race ./memory/manager -count=1 unit-test: name: Unit test diff --git a/memory/manager/manager.go b/memory/manager/manager.go index 3695d8520..cdf95c5d3 100644 --- a/memory/manager/manager.go +++ b/memory/manager/manager.go @@ -267,10 +267,10 @@ func (m *MemoryManager) Deactivate(vmID string) error { return errors.New("VM not activated") } - select { - case state.quitCh <- 0: - default: - } + state.stopPolling() + state.waitForPoller() + state.closeWakeFD() + if err := state.unmapGuestMemory(); err != nil { logger.Error("Failed to munmap guest memory") return err diff --git a/memory/manager/manager_test.go b/memory/manager/manager_test.go index abb38a060..8882cb0f7 100644 --- a/memory/manager/manager_test.go +++ b/memory/manager/manager_test.go @@ -23,7 +23,6 @@ package manager import ( - "encoding/binary" "encoding/json" "errors" "os" @@ -169,8 +168,6 @@ func TestMemoryManagerActivateReceivesFirecrackerMappings(t *testing.T) { t.Fatalf("validateGuestMemory mapped memory returned error: %v", err) } - signalEventFD(t, uffdStandIn) - if err := manager.Deactivate(vmID); err != nil { t.Fatalf("Deactivate returned error: %v", err) } @@ -223,16 +220,6 @@ func testEventFD(t *testing.T) *os.File { return file } -func signalEventFD(t *testing.T, file *os.File) { - t.Helper() - - var buf [8]byte - binary.LittleEndian.PutUint64(buf[:], 1) - if _, err := unix.Write(int(file.Fd()), buf[:]); err != nil { - t.Fatalf("unix.Write(eventfd) returned error: %v", err) - } -} - func prepareGuestMemoryFile(t *testing.T, guestFileName string, size int) { t.Helper() diff --git a/memory/manager/snapshot_state.go b/memory/manager/snapshot_state.go index 844f0da2c..44db9db31 100644 --- a/memory/manager/snapshot_state.go +++ b/memory/manager/snapshot_state.go @@ -155,7 +155,9 @@ type SnapshotState struct { userFaultFD *os.File guestRegionMappings []GuestRegionUffdMapping epfd int + wakeFD int quitCh chan int + pollDoneCh chan struct{} // to indicate whether the instance has even been activated. this is to // get around cases where offload is called for the first time @@ -189,7 +191,9 @@ func NewSnapshotState(cfg SnapshotStateCfg) *SnapshotState { func (s *SnapshotState) setupStateOnActivate() { s.isActive = true s.isEverActivated = true + s.wakeFD = -1 s.quitCh = make(chan int, 1) + s.pollDoneCh = make(chan struct{}) if s.metricsModeOn { s.uniqueNum = 0 @@ -388,7 +392,9 @@ func (s *SnapshotState) fetchState() error { func (s *SnapshotState) pollUserPageFaults(readyCh chan error) { logger := log.WithFields(log.Fields{"vmID": s.VMID}) - var events [1]syscall.EpollEvent + var events [2]syscall.EpollEvent + + defer close(s.pollDoneCh) if err := s.registerEpoller(); err != nil { readyCh <- err @@ -424,10 +430,21 @@ func (s *SnapshotState) pollUserPageFaults(readyCh chan error) { continue } + select { + case <-s.quitCh: + logger.Debug("Handler received a signal to quit") + return + default: + } + for i := 0; i < nevents; i++ { event := events[i] fd := int(event.Fd) + if fd == s.wakeFD { + logger.Debug("Handler received wakeup event") + return + } stateFd := int(s.userFaultFD.Fd()) @@ -508,9 +525,61 @@ func (s *SnapshotState) registerEpoller() error { return err } + s.wakeFD, err = unix.Eventfd(0, unix.EFD_CLOEXEC|unix.EFD_NONBLOCK) + if err != nil { + _ = syscall.Close(s.epfd) + logger.Errorf("Failed to create UFFD wake fd %v", err) + return err + } + + event.Fd = int32(s.wakeFD) + if err := syscall.EpollCtl( + s.epfd, + syscall.EPOLL_CTL_ADD, + s.wakeFD, + &event, + ); err != nil { + _ = unix.Close(s.wakeFD) + _ = syscall.Close(s.epfd) + logger.Errorf("Failed to subscribe UFFD wake fd %v", err) + return err + } + return nil } +func (s *SnapshotState) stopPolling() { + select { + case s.quitCh <- 0: + default: + } + + if s.wakeFD < 0 { + return + } + + var wake [8]byte + binary.LittleEndian.PutUint64(wake[:], 1) + if _, err := unix.Write(s.wakeFD, wake[:]); err != nil && + !errors.Is(err, syscall.EBADF) && + !errors.Is(err, syscall.EAGAIN) { + log.WithError(err).Debug("Failed to wake UFFD poller") + } +} + +func (s *SnapshotState) waitForPoller() { + if s.pollDoneCh != nil { + <-s.pollDoneCh + } +} + +func (s *SnapshotState) closeWakeFD() { + if s.wakeFD >= 0 { + _ = unix.Close(s.wakeFD) + s.wakeFD = -1 + } +} + func (s *SnapshotState) servePageFault(fd int, address uint64) error { var tStart time.Time From fe646bcf9d2b149b578e1b2a8c2e4a46245926b6 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Sun, 5 Jul 2026 18:57:22 +0800 Subject: [PATCH 11/21] Use File memory backend for snapshot load --- ctriface/iface.go | 16 ++++++++++------ ctriface/proto_compat_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/ctriface/iface.go b/ctriface/iface.go index c0f8b8e6e..4dd97d48c 100644 --- a/ctriface/iface.go +++ b/ctriface/iface.go @@ -505,6 +505,14 @@ func logSnapshotLoadFailure(logger *log.Entry, snap *snapshotting.Snapshot, conf }).Error("failed to load snapshot of the VM") } +func configureSnapshotMemoryBackend(conf *proto.CreateVMRequest, backendType, backendPath string) { + conf.MemFilePath = "" + conf.MemBackend = &proto.MemoryBackend{ + BackendType: backendType, + BackendPath: backendPath, + } +} + // LoadSnapshot Loads a snapshot of a VM func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snapshotting.Snapshot) (_ *StartVMResponse, _ *metrics.Metric, retErr error) { var ( @@ -536,7 +544,7 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap conf := o.getVMConfig(vm) conf.LoadSnapshot = true conf.SnapshotPath = snap.GetSnapshotFilePath() - conf.MemFilePath = snap.GetMemFilePath() + configureSnapshotMemoryBackend(conf, "File", snap.GetMemFilePath()) uffdSock := filepath.Join(o.getVMBaseDir(vmID), "uffd.sock") if o.snapshotter == "devmapper" { @@ -596,11 +604,7 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap } if o.GetUPFEnabled() { - conf.MemFilePath = "" - conf.MemBackend = &proto.MemoryBackend{ - BackendType: "Uffd", - BackendPath: uffdSock, - } + configureSnapshotMemoryBackend(conf, "Uffd", uffdSock) if err := o.memoryManager.PrepareSnapshotLoad(manager.SnapshotStateCfg{ VMID: vmID, diff --git a/ctriface/proto_compat_test.go b/ctriface/proto_compat_test.go index 692fd0f63..b7dbe0b0d 100644 --- a/ctriface/proto_compat_test.go +++ b/ctriface/proto_compat_test.go @@ -27,3 +27,33 @@ func TestCreateVMRequestWithMemoryBackendMarshalsForTTRPC(t *testing.T) { require.Equal(t, "Uffd", decoded.GetMemBackend().GetBackendType()) require.Equal(t, "/tmp/vhive-upf.sock", decoded.GetMemBackend().GetBackendPath()) } + +func TestConfigureSnapshotMemoryBackendClearsLegacyMemFilePath(t *testing.T) { + req := &fcproto.CreateVMRequest{ + MemFilePath: "/fccd/snapshots/myrev/mem_file", + } + + configureSnapshotMemoryBackend(req, "File", "/fccd/snapshots/myrev/mem_file") + + require.Empty(t, req.GetMemFilePath()) + require.Equal(t, "File", req.GetMemBackend().GetBackendType()) + require.Equal(t, "/fccd/snapshots/myrev/mem_file", req.GetMemBackend().GetBackendPath()) +} + +func TestCreateVMRequestWithFileMemoryBackendMarshalsForTTRPC(t *testing.T) { + req := &fcproto.CreateVMRequest{ + VMID: "vm-with-file-backend", + LoadSnapshot: true, + } + configureSnapshotMemoryBackend(req, "File", "/fccd/snapshots/myrev/mem_file") + + payload, err := googleproto.Marshal(req) + require.NoError(t, err) + require.NotEmpty(t, payload) + + var decoded fcproto.CreateVMRequest + require.NoError(t, googleproto.Unmarshal(payload, &decoded)) + require.Empty(t, decoded.GetMemFilePath()) + require.Equal(t, "File", decoded.GetMemBackend().GetBackendType()) + require.Equal(t, "/fccd/snapshots/myrev/mem_file", decoded.GetMemBackend().GetBackendPath()) +} From df2fba2c540866fde6dc381cc5a8ad6afdc16718 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Thu, 9 Jul 2026 15:05:22 +0800 Subject: [PATCH 12/21] Align memory manager checkout action --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5c8cf5fba..86525bb5d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Check out code into the Go module directory - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Go version in go.mod file uses: actions/setup-go@v6 From ab2099af46654683562a4c91b5535a7434cc1afe Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Thu, 9 Jul 2026 15:21:51 +0800 Subject: [PATCH 13/21] fix: resolve CI lint failures Signed-off-by: Liquidzk <492928615@qq.com> --- ctriface/orch.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ctriface/orch.go b/ctriface/orch.go index 6eadd10eb..a36cf8089 100644 --- a/ctriface/orch.go +++ b/ctriface/orch.go @@ -234,10 +234,6 @@ func (o *Orchestrator) getMemoryFile(vmID string) string { return filepath.Join(o.getVMBaseDir(vmID), "mem_file") } -func (o *Orchestrator) getWorkingSetFile(vmID string) string { - return filepath.Join(o.getVMBaseDir(vmID), "working_set_pages") -} - func (o *Orchestrator) getVMBaseDir(vmID string) string { return filepath.Join(o.snapshotsDir, vmID) } From 1cc99b6f6cf6b7241beb4a952f0fe7aac9d982eb Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Thu, 9 Jul 2026 15:28:25 +0800 Subject: [PATCH 14/21] ci: lint pull request head commit Signed-off-by: Liquidzk <492928615@qq.com> --- .github/workflows/linters.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 8481c919e..b1768d96e 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -22,6 +22,7 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-node@v6 with: node-version: '20' From e9a350e72f3c4cd0f4b9a140939f919f534f31b8 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Thu, 9 Jul 2026 21:30:20 +0800 Subject: [PATCH 15/21] ci: rerun pull request checks Signed-off-by: Liquidzk <492928615@qq.com> From 21f105ebc999e88c0ff06a129a44109b0a729c35 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Mon, 13 Jul 2026 10:41:26 +0800 Subject: [PATCH 16/21] Modify unit_test and restore unused working sets Signed-off-by: Liquidzk <492928615@qq.com> --- .github/workflows/unit_tests.yml | 30 +-- ctriface/iface.go | 18 +- ctriface/iface_test.go | 10 + ctriface/orch.go | 5 + ctriface/orch_options.go | 13 +- memory/manager/manager.go | 85 ++++++- memory/manager/manager_test.go | 59 +++++ memory/manager/snapshot_state.go | 331 +++++++++++++++++++++++++- memory/manager/snapshot_state_test.go | 83 +++++++ memory/manager/trace.go | 210 ++++++++++++++++ memory/manager/user_page_faults.h | 2 + vhive.go | 4 + vhive_test.go | 4 + 13 files changed, 808 insertions(+), 46 deletions(-) create mode 100644 memory/manager/trace.go diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 86525bb5d..51bbf8bd7 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -20,33 +20,13 @@ env: GO111MODULE: on jobs: - memory-manager-unit-test: - name: "Unit tests: memory manager" - runs-on: ubuntu-24.04 - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v7 - - - name: Set up Go version in go.mod file - uses: actions/setup-go@v6 - with: - go-version-file: ${{ github.workspace }}/go.mod - cache-dependency-path: | - **/go.sum - **/go.mod - - - name: Run memory manager tests - run: | - go test ./memory/manager - go test -race ./memory/manager -count=1 - unit-test: name: Unit test runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: - module: [misc, networking, snapshotting] + module: [misc, networking, snapshotting, memory/manager] steps: - name: Check out code into the Go module directory uses: actions/checkout@v7 @@ -84,8 +64,12 @@ jobs: AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} run: | - make -C $MODULE test - make -C $MODULE test-man + if [[ "$MODULE" == "memory/manager" ]]; then + go test -race ./memory/manager -count=1 + else + make -C "$MODULE" test + make -C "$MODULE" test-man + fi profile-unit-test: if: false diff --git a/ctriface/iface.go b/ctriface/iface.go index 4dd97d48c..1933c6b64 100644 --- a/ctriface/iface.go +++ b/ctriface/iface.go @@ -248,11 +248,13 @@ func (o *Orchestrator) StartVMWithEnvironment(ctx context.Context, vmID, imageNa logger.Debug("Registering VM with the memory manager") stateCfg := manager.SnapshotStateCfg{ - VMID: vmID, - GuestMemPath: o.getMemoryFile(vmID), - BaseDir: o.getVMBaseDir(vmID), - GuestMemSize: int(conf.MachineCfg.MemSizeMib) * 1024 * 1024, - VMMStatePath: o.getSnapshotFile(vmID), + VMID: vmID, + GuestMemPath: o.getMemoryFile(vmID), + BaseDir: o.getVMBaseDir(vmID), + GuestMemSize: int(conf.MachineCfg.MemSizeMib) * 1024 * 1024, + IsLazyMode: o.isLazyMode, + VMMStatePath: o.getSnapshotFile(vmID), + WorkingSetPath: o.getWorkingSetFile(vmID), } if err := o.memoryManager.RegisterVM(stateCfg); err != nil { return nil, nil, errors.Wrap(err, "failed to register VM with memory manager") @@ -515,6 +517,10 @@ func configureSnapshotMemoryBackend(conf *proto.CreateVMRequest, backendType, ba // LoadSnapshot Loads a snapshot of a VM func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snapshotting.Snapshot) (_ *StartVMResponse, _ *metrics.Metric, retErr error) { + if err := o.validateUPFMode(); err != nil { + return nil, nil, err + } + var ( loadSnapshotMetric = metrics.NewMetric() tStart time.Time @@ -613,6 +619,8 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap InstanceSockAddr: uffdSock, BaseDir: o.getVMBaseDir(vmID), GuestMemSize: int(conf.MachineCfg.MemSizeMib) * 1024 * 1024, + IsLazyMode: o.isLazyMode, + WorkingSetPath: o.getWorkingSetFile(vmID), }); err != nil { return nil, nil, err } diff --git a/ctriface/iface_test.go b/ctriface/iface_test.go index 6a7b0eb32..52d30a26c 100644 --- a/ctriface/iface_test.go +++ b/ctriface/iface_test.go @@ -54,6 +54,16 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } +func TestValidateUPFModeRequiresLazyMode(t *testing.T) { + orch := &Orchestrator{} + WithUPF(true)(orch) + + require.ErrorIs(t, orch.validateUPFMode(), errUPFRequiresLazyMode) + + WithLazyMode(true)(orch) + require.NoError(t, orch.validateUPFMode()) +} + func TestStartSnapStopLoad(t *testing.T) { log.SetFormatter(&log.TextFormatter{ TimestampFormat: ctrdlog.RFC3339NanoFixed, diff --git a/ctriface/orch.go b/ctriface/orch.go index a36cf8089..b6ec2be04 100644 --- a/ctriface/orch.go +++ b/ctriface/orch.go @@ -100,6 +100,7 @@ type Orchestrator struct { // store *skv.KVStore snapshotsEnabled bool isUPFEnabled bool + isLazyMode bool snapshotsDir string isMetricsMode bool netPoolSize int @@ -234,6 +235,10 @@ func (o *Orchestrator) getMemoryFile(vmID string) string { return filepath.Join(o.getVMBaseDir(vmID), "mem_file") } +func (o *Orchestrator) getWorkingSetFile(vmID string) string { + return filepath.Join(o.getVMBaseDir(vmID), "working_set_pages") +} + func (o *Orchestrator) getVMBaseDir(vmID string) string { return filepath.Join(o.snapshotsDir, vmID) } diff --git a/ctriface/orch_options.go b/ctriface/orch_options.go index 70143f75d..980b090bb 100644 --- a/ctriface/orch_options.go +++ b/ctriface/orch_options.go @@ -24,9 +24,12 @@ package ctriface import ( "encoding/json" + "errors" "fmt" ) +var errUPFRequiresLazyMode = errors.New("UPF currently requires lazy mode") + // OrchestratorOption Options to pass to Orchestrator type OrchestratorOption func(*Orchestrator) @@ -62,10 +65,18 @@ func WithSnapshotsDir(snapshotsDir string) OrchestratorOption { } } -// WithLazyMode is kept for compatibility with legacy callers. +// WithLazyMode Sets the lazy paging mode on or off. func WithLazyMode(isLazyMode bool) OrchestratorOption { return func(o *Orchestrator) { + o.isLazyMode = isLazyMode + } +} + +func (o *Orchestrator) validateUPFMode() error { + if o.isUPFEnabled && !o.isLazyMode { + return errUPFRequiresLazyMode } + return nil } // WithMetricsMode Sets the metrics mode diff --git a/memory/manager/manager.go b/memory/manager/manager.go index cdf95c5d3..d007eed9f 100644 --- a/memory/manager/manager.go +++ b/memory/manager/manager.go @@ -27,7 +27,9 @@ import ( "errors" "fmt" "os" + "strconv" "sync" + "time" "github.com/vhive-serverless/vhive/metrics" "gonum.org/v1/gonum/stat" @@ -37,6 +39,8 @@ import ( const ( serveUniqueMetric = "ServeUnique" + installWSMetric = "InstallWS" + fetchStateMetric = "FetchState" ) // MemoryManagerCfg Global config of the manager @@ -98,11 +102,10 @@ func (m *MemoryManager) PrepareSnapshotLoad(cfg SnapshotStateCfg) error { } cfg.metricsModeOn = m.MetricsModeOn - nextState := NewSnapshotState(cfg) state, ok := m.instances[vmID] if !ok { - m.instances[vmID] = nextState + m.instances[vmID] = NewSnapshotState(cfg) return nil } if state.isActive { @@ -110,9 +113,10 @@ func (m *MemoryManager) PrepareSnapshotLoad(cfg SnapshotStateCfg) error { } if state.userFaultFD != nil { _ = state.userFaultFD.Close() + state.userFaultFD = nil } - *state = *nextState + state.refreshSnapshotLoad(cfg) return nil } @@ -218,8 +222,9 @@ func (m *MemoryManager) FetchState(vmID string) error { logger.Debug("Fetching state in the memory manager") var ( - ok bool - state *SnapshotState + ok bool + state *SnapshotState + tStart time.Time ) m.Lock() @@ -233,7 +238,18 @@ func (m *MemoryManager) FetchState(vmID string) error { m.Unlock() - return state.fetchState() + if state.metricsModeOn && state.currentMetric == nil { + state.currentMetric = metrics.NewMetric() + } + if state.metricsModeOn && state.isRecordReady && !state.IsLazyMode { + tStart = time.Now() + } + + err := state.fetchState() + if err == nil && !tStart.IsZero() { + state.currentMetric.MetricMap[fetchStateMetric] = metrics.ToUS(time.Since(tStart)) + } + return err } // Deactivate Removes the epoller which serves page faults for the VM @@ -282,6 +298,17 @@ func (m *MemoryManager) Deactivate(vmID string) error { defer func() { _ = state.userFaultFD.Close() }() } + if !state.isRecordReady && !state.IsLazyMode { + pageSize, err := guestMappingPageSize(state.guestRegionMappings) + if err != nil { + return err + } + if err := state.trace.ProcessRecord(state.GuestMemPath, state.WorkingSetPath, pageSize); err != nil { + return err + } + } + + state.isRecordReady = true state.isActive = false return nil @@ -319,7 +346,11 @@ func (m *MemoryManager) DumpUPFPageStats(vmID, functionName, metricsOutFilePath return errors.New("metrics mode is not on") } - statHeader, stats = getUPFHeaderStats(state, functionName) + if state.IsLazyMode { + statHeader, stats = getLazyHeaderStats(state, functionName) + } else { + statHeader, stats = getRecRepHeaderStats(state, functionName) + } return writeUPFPageStats(metricsOutFilePath, statHeader, stats) } @@ -385,10 +416,42 @@ func (m *MemoryManager) GetUPFLatencyStats(vmID string) ([]*metrics.Metric, erro return state.latencyMetrics, nil } -func getUPFHeaderStats(state *SnapshotState, functionName string) ([]string, []string) { +func getLazyHeaderStats(state *SnapshotState, functionName string) ([]string, []string) { + header := []string{ + "FuncName", + "RecPages", + "RepPages", + "StdDev", + "Reused", + "StdDev", + "Unique", + "StdDev", + } + + uniqueMean, uniqueStd := stat.MeanStdDev(state.uniquePFServed, nil) + totalMean, totalStd := stat.MeanStdDev(state.totalPFServed, nil) + reusedMean, reusedStd := stat.MeanStdDev(state.reusedPFServed, nil) + + stats := []string{ + functionName, + strconv.Itoa(len(state.trace.trace)), + strconv.Itoa(int(totalMean)), + fmt.Sprintf("%.1f", totalStd), + strconv.Itoa(int(reusedMean)), + fmt.Sprintf("%.1f", reusedStd), + strconv.Itoa(int(uniqueMean)), + fmt.Sprintf("%.1f", uniqueStd), + } + + return header, stats +} + +func getRecRepHeaderStats(state *SnapshotState, functionName string) ([]string, []string) { header := []string{ "FuncName", - "ServedPages", + "RecPages", + "RecRegions", + "Unique", "StdDev", } @@ -396,7 +459,9 @@ func getUPFHeaderStats(state *SnapshotState, functionName string) ([]string, []s stats := []string{ functionName, - fmt.Sprintf("%.0f", uniqueMean), + strconv.Itoa(len(state.trace.trace)), + strconv.Itoa(len(state.trace.regions)), + strconv.Itoa(int(uniqueMean)), fmt.Sprintf("%.1f", uniqueStd), } diff --git a/memory/manager/manager_test.go b/memory/manager/manager_test.go index 8882cb0f7..4ac83b2b3 100644 --- a/memory/manager/manager_test.go +++ b/memory/manager/manager_test.go @@ -102,6 +102,65 @@ func TestMemoryManagerRegisterFetchPrepareDeregister(t *testing.T) { } } +func TestPrepareSnapshotLoadPreservesWorkingSetState(t *testing.T) { + baseDir := t.TempDir() + vmID := "vm-prepare-ws" + guestMemPath := filepath.Join(baseDir, "guest_mem") + vmmStatePath := filepath.Join(baseDir, "state") + workingSetPath := filepath.Join(baseDir, "working_set_pages") + + prepareGuestMemoryFile(t, guestMemPath, 2*os.Getpagesize()) + writeTestFile(t, vmmStatePath, "state") + + manager := NewMemoryManager(MemoryManagerCfg{}) + cfg := SnapshotStateCfg{ + VMID: vmID, + BaseDir: baseDir, + VMMStatePath: vmmStatePath, + GuestMemPath: guestMemPath, + WorkingSetPath: workingSetPath, + GuestMemSize: 2 * os.Getpagesize(), + } + if err := manager.RegisterVM(cfg); err != nil { + t.Fatalf("RegisterVM returned error: %v", err) + } + + state := manager.instances[vmID] + state.trace.AppendRecord(Record{offset: 0}) + state.isRecordReady = true + trace := state.trace + + nextVMMStatePath := filepath.Join(baseDir, "next_state") + nextSocketPath := filepath.Join(baseDir, "next_uffd.sock") + writeTestFile(t, nextVMMStatePath, "next-state") + + nextCfg := cfg + nextCfg.VMMStatePath = nextVMMStatePath + nextCfg.InstanceSockAddr = nextSocketPath + nextCfg.IsLazyMode = true + + if err := manager.PrepareSnapshotLoad(nextCfg); err != nil { + t.Fatalf("PrepareSnapshotLoad returned error: %v", err) + } + + got := manager.instances[vmID] + if got.trace != trace { + t.Fatal("PrepareSnapshotLoad replaced trace state") + } + if !got.isRecordReady { + t.Fatal("PrepareSnapshotLoad cleared isRecordReady") + } + if got.VMMStatePath != nextVMMStatePath { + t.Fatalf("VMMStatePath = %q, want %q", got.VMMStatePath, nextVMMStatePath) + } + if got.InstanceSockAddr != nextSocketPath { + t.Fatalf("InstanceSockAddr = %q, want %q", got.InstanceSockAddr, nextSocketPath) + } + if !got.IsLazyMode { + t.Fatal("PrepareSnapshotLoad did not update IsLazyMode") + } +} + func TestMemoryManagerActivateReceivesFirecrackerMappings(t *testing.T) { baseDir := t.TempDir() vmID := "vm-activate" diff --git a/memory/manager/snapshot_state.go b/memory/manager/snapshot_state.go index 44db9db31..c8809d325 100644 --- a/memory/manager/snapshot_state.go +++ b/memory/manager/snapshot_state.go @@ -32,10 +32,13 @@ import ( "encoding/json" "errors" "fmt" + "io" "math" "net" "os" "path/filepath" + "sort" + "sync" "syscall" "time" @@ -101,6 +104,22 @@ func guestMemoryOffsetForFaultPage(region GuestRegionUffdMapping, faultPageAddr return region.Offset + regionOffset, nil } +func guestAddressForMemoryOffset(region GuestRegionUffdMapping, offset uint64) (uint64, error) { + if region.PageSize == 0 { + return 0, errInvalidGuestRegionPageSize + } + if !regionContainsGuestMemoryOffset(region, offset) { + return 0, fmt.Errorf("%w: %#x", errGuestRegionNotFound, offset) + } + + regionOffset := offset - region.Offset + if region.BaseHostVirtAddr > math.MaxUint64-regionOffset { + return 0, fmt.Errorf("guest address overflow for memory offset %#x", offset) + } + + return region.BaseHostVirtAddr + regionOffset, nil +} + func pageFaultCopyArgsForFault(regions []GuestRegionUffdMapping, faultAddr uint64) (pageFaultCopyArgs, error) { for _, region := range regions { faultPageAddr, err := pageAlignFaultAddress(faultAddr, region) @@ -127,6 +146,27 @@ func pageFaultCopyArgsForFault(regions []GuestRegionUffdMapping, faultAddr uint6 return pageFaultCopyArgs{}, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultAddr) } +func pageFaultCopyArgsForGuestOffset(regions []GuestRegionUffdMapping, offset uint64, mode uint64) (pageFaultCopyArgs, error) { + for _, region := range regions { + if !regionContainsGuestMemoryOffset(region, offset) { + continue + } + dstAddr, err := guestAddressForMemoryOffset(region, offset) + if err != nil { + return pageFaultCopyArgs{}, err + } + + return pageFaultCopyArgs{ + srcOffset: offset, + dstAddr: dstAddr, + copyLen: region.PageSize, + copyMode: mode, + }, nil + } + + return pageFaultCopyArgs{}, fmt.Errorf("%w: %#x", errGuestRegionNotFound, offset) +} + func regionContainsFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) bool { if region.Size == 0 || faultPageAddr < region.BaseHostVirtAddr { return false @@ -135,15 +175,44 @@ func regionContainsFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64 return faultPageAddr-region.BaseHostVirtAddr < region.Size } +func regionContainsGuestMemoryOffset(region GuestRegionUffdMapping, offset uint64) bool { + if region.Size == 0 || offset < region.Offset { + return false + } + + return offset-region.Offset < region.Size +} + +func guestMappingPageSize(regions []GuestRegionUffdMapping) (uint64, error) { + var pageSize uint64 + for _, region := range regions { + if region.PageSize == 0 { + return 0, errInvalidGuestRegionPageSize + } + if pageSize == 0 { + pageSize = region.PageSize + continue + } + if pageSize != region.PageSize { + return 0, errors.New("mixed guest region page sizes are not supported for working-set replay") + } + } + if pageSize == 0 { + return 0, errNoGuestRegionMappings + } + return pageSize, nil +} + // SnapshotStateCfg Config to initialize SnapshotState type SnapshotStateCfg struct { VMID string - VMMStatePath, GuestMemPath string + VMMStatePath, GuestMemPath, WorkingSetPath string InstanceSockAddr string BaseDir string // base directory for the instance MetricsPath string // path to csv file where the metrics should be stored + IsLazyMode bool GuestMemSize int metricsModeOn bool } @@ -152,8 +221,10 @@ type SnapshotStateCfg struct { // of the VM. type SnapshotState struct { SnapshotStateCfg + firstPageFaultOnce *sync.Once userFaultFD *os.File guestRegionMappings []GuestRegionUffdMapping + trace *Trace epfd int wakeFD int quitCh chan int @@ -165,12 +236,18 @@ type SnapshotState struct { // for sanity checking on deactivate/activate isActive bool - guestMem []byte + isRecordReady bool + + guestMem []byte + workingSet []byte // Stats + totalPFServed []float64 uniquePFServed []float64 + reusedPFServed []float64 latencyMetrics []*metrics.Metric + replayedNum int uniqueNum int currentMetric *metrics.Metric } @@ -178,26 +255,95 @@ type SnapshotState struct { // NewSnapshotState Initializes a snapshot state func NewSnapshotState(cfg SnapshotStateCfg) *SnapshotState { s := new(SnapshotState) + cfg = normalizeSnapshotStateCfg(cfg) s.SnapshotStateCfg = cfg + s.trace = initTrace(s.getTraceFile()) if s.metricsModeOn { + s.totalPFServed = make([]float64, 0) s.uniquePFServed = make([]float64, 0) + s.reusedPFServed = make([]float64, 0) s.latencyMetrics = make([]*metrics.Metric, 0) } return s } +func normalizeSnapshotStateCfg(cfg SnapshotStateCfg) SnapshotStateCfg { + if cfg.WorkingSetPath == "" && cfg.BaseDir != "" { + cfg.WorkingSetPath = filepath.Join(cfg.BaseDir, "working_set_pages") + } + return cfg +} + +func (s *SnapshotState) refreshSnapshotLoad(cfg SnapshotStateCfg) { + cfg = normalizeSnapshotStateCfg(cfg) + + trace := s.trace + if trace == nil { + trace = initTrace(filepath.Join(cfg.BaseDir, "trace")) + } else { + trace.traceFileName = filepath.Join(cfg.BaseDir, "trace") + } + isRecordReady := s.isRecordReady + isEverActivated := s.isEverActivated + totalPFServed := s.totalPFServed + uniquePFServed := s.uniquePFServed + reusedPFServed := s.reusedPFServed + latencyMetrics := s.latencyMetrics + + s.SnapshotStateCfg = cfg + s.firstPageFaultOnce = nil + s.userFaultFD = nil + s.guestRegionMappings = nil + s.trace = trace + s.epfd = 0 + s.wakeFD = -1 + s.quitCh = nil + s.pollDoneCh = nil + s.isEverActivated = isEverActivated + s.isActive = false + s.isRecordReady = isRecordReady + s.guestMem = nil + s.workingSet = nil + s.totalPFServed = totalPFServed + s.uniquePFServed = uniquePFServed + s.reusedPFServed = reusedPFServed + s.latencyMetrics = latencyMetrics + s.replayedNum = 0 + s.uniqueNum = 0 + s.currentMetric = nil + + if s.metricsModeOn { + if s.totalPFServed == nil { + s.totalPFServed = make([]float64, 0) + } + if s.uniquePFServed == nil { + s.uniquePFServed = make([]float64, 0) + } + if s.reusedPFServed == nil { + s.reusedPFServed = make([]float64, 0) + } + if s.latencyMetrics == nil { + s.latencyMetrics = make([]*metrics.Metric, 0) + } + } +} + func (s *SnapshotState) setupStateOnActivate() { s.isActive = true s.isEverActivated = true + s.firstPageFaultOnce = new(sync.Once) s.wakeFD = -1 s.quitCh = make(chan int, 1) s.pollDoneCh = make(chan struct{}) if s.metricsModeOn { s.uniqueNum = 0 - s.currentMetric = metrics.NewMetric() + s.replayedNum = 0 + if s.currentMetric == nil { + s.currentMetric = metrics.NewMetric() + } } } @@ -347,10 +493,24 @@ func removeStaleUffdSocket(socketPath string) error { } func (s *SnapshotState) processMetrics() { - if s.metricsModeOn && s.currentMetric != nil { + if !s.metricsModeOn || s.currentMetric == nil { + return + } + + if s.isRecordReady { + if s.IsLazyMode { + s.totalPFServed = append(s.totalPFServed, float64(s.replayedNum)) + s.reusedPFServed = append(s.reusedPFServed, float64(s.replayedNum-s.uniqueNum)) + } + s.uniquePFServed = append(s.uniquePFServed, float64(s.uniqueNum)) s.latencyMetrics = append(s.latencyMetrics, s.currentMetric) } + s.currentMetric = nil +} + +func (s *SnapshotState) getTraceFile() string { + return filepath.Join(s.BaseDir, "trace") } func (s *SnapshotState) mapGuestMemory() error { @@ -379,16 +539,76 @@ func (s *SnapshotState) unmapGuestMemory() error { return nil } -// fetchState verifies the VMM state file before snapshot activation. +// fetchState verifies snapshot state and loads the replay working set when ready. func (s *SnapshotState) fetchState() error { if _, err := os.ReadFile(s.VMMStatePath); err != nil { log.Errorf("Failed to fetch VMM state: %v\n", err) return err } + if s.isRecordReady && !s.IsLazyMode { + return s.fetchWorkingSet() + } + return nil } +func (s *SnapshotState) fetchWorkingSet() error { + pageSize := s.trace.pageSize + if pageSize == 0 { + pageSize = uint64(os.Getpagesize()) + } + + size := uint64(len(s.trace.trace)) * pageSize + if size > uint64(int(^uint(0)>>1)) { + return fmt.Errorf("working set too large: %#x", size) + } + if size == 0 { + s.workingSet = nil + return nil + } + + f, err := os.Open(s.WorkingSetPath) + if err != nil { + log.Errorf("Failed to open the working set file: %v\n", err) + return err + } + defer func() { _ = f.Close() }() + + s.workingSet = AlignedBlock(int(size)) + n, err := io.ReadFull(f, s.workingSet) + if err != nil { + log.Errorf("Reading working set file failed: %v\n", err) + return err + } + if n != len(s.workingSet) { + return io.ErrUnexpectedEOF + } + + log.Debug("Fetched the entire working set") + return nil +} + +// alignment returns alignment of the block in memory with reference to alignSize. +func alignment(block []byte, alignSize int) int { + return int(uintptr(unsafe.Pointer(&block[0])) & uintptr(alignSize-1)) +} + +// AlignedBlock returns a []byte aligned to the host page size. +func AlignedBlock(blockSize int) []byte { + if blockSize == 0 { + return nil + } + + alignSize := os.Getpagesize() + block := make([]byte, blockSize+alignSize) + + if a := alignment(block, alignSize); a != 0 { + block = block[alignSize-a:] + } + return block[:blockSize] +} + func (s *SnapshotState) pollUserPageFaults(readyCh chan error) { logger := log.WithFields(log.Fields{"vmID": s.VMID}) @@ -581,20 +801,66 @@ func (s *SnapshotState) closeWakeFD() { } func (s *SnapshotState) servePageFault(fd int, address uint64) error { - var tStart time.Time + var ( + tStart time.Time + workingSetInstalled bool + ) copyArgs, err := pageFaultCopyArgsForFault(s.guestRegionMappings, address) if err != nil { return err } + rec := Record{offset: copyArgs.srcOffset} + if s.firstPageFaultOnce != nil { + s.firstPageFaultOnce.Do(func() { + if !s.isRecordReady || s.IsLazyMode { + return + } + + if s.metricsModeOn { + tStart = time.Now() + } + err = s.installWorkingSetPages(fd, copyArgs.dstAddr, copyArgs.copyLen) + if err != nil { + return + } + if s.metricsModeOn { + s.currentMetric.MetricMap[installWSMetric] = metrics.ToUS(time.Since(tStart)) + } + workingSetInstalled = true + }) + if err != nil { + return err + } + } + + if workingSetInstalled && s.trace.containsRecord(rec) { + return nil + } + src, err := guestMemPointer(s.guestMem, copyArgs.srcOffset, copyArgs.copyLen) if err != nil { return err } + if !s.isRecordReady { + s.trace.AppendRecord(rec) + } else { + log.Debug("Serving a page that is missing from the working set") + } + if s.metricsModeOn { - s.uniqueNum++ + if s.isRecordReady { + if s.IsLazyMode { + if !s.trace.containsRecord(rec) { + s.uniqueNum++ + } + s.replayedNum++ + } else { + s.uniqueNum++ + } + } tStart = time.Now() } @@ -607,6 +873,48 @@ func (s *SnapshotState) servePageFault(fd int, address uint64) error { return err } +func (s *SnapshotState) installWorkingSetPages(fd int, faultPageAddr, pageSize uint64) error { + if len(s.workingSet) == 0 || len(s.trace.regions) == 0 { + return nil + } + if s.trace.pageSize != 0 { + pageSize = s.trace.pageSize + } + + keys := make([]uint64, 0, len(s.trace.regions)) + for offset := range s.trace.regions { + keys = append(keys, offset) + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + + var workingSetOffset uint64 + for _, regionOffset := range keys { + regLength := s.trace.regions[regionOffset] + for i := 0; i < regLength; i++ { + pageOffset := regionOffset + uint64(i)*pageSize + copyArgs, err := pageFaultCopyArgsForGuestOffset( + s.guestRegionMappings, + pageOffset, + uint64(C.const_UFFDIO_COPY_MODE_DONTWAKE), + ) + if err != nil { + return err + } + + src, err := guestMemPointer(s.workingSet, workingSetOffset, copyArgs.copyLen) + if err != nil { + return err + } + if err := installRegionBytes(fd, src, copyArgs.dstAddr, copyArgs.copyMode, copyArgs.copyLen); err != nil { + return err + } + workingSetOffset += copyArgs.copyLen + } + } + + return wake(fd, faultPageAddr, pageSize) +} + func installRegionBytes(fd int, src, dst, mode, length uint64) error { cUC := C.struct_uffdio_copy{ mode: C.ulonglong(mode), @@ -655,6 +963,15 @@ func ioctl(fd uintptr, request int, argp unsafe.Pointer) error { return nil } +func wake(fd int, startAddress, length uint64) error { + cUR := C.struct_uffdio_range{ + start: C.ulonglong(startAddress), + len: C.ulonglong(length), + } + + return ioctl(uintptr(fd), int(C.const_UFFDIO_WAKE), unsafe.Pointer(&cUR)) +} + func sizeOfUFFDMsg() int { return C.sizeof_struct_uffd_msg } diff --git a/memory/manager/snapshot_state_test.go b/memory/manager/snapshot_state_test.go index 4a311d289..df814d225 100644 --- a/memory/manager/snapshot_state_test.go +++ b/memory/manager/snapshot_state_test.go @@ -165,6 +165,89 @@ func TestPageFaultCopyArgsForFaultZeroPageSize(t *testing.T) { } } +func TestPageFaultCopyArgsForGuestOffset(t *testing.T) { + regions := []GuestRegionUffdMapping{ + { + BaseHostVirtAddr: 0x100000, + Size: 0x2000, + PageSize: 0x1000, + }, + { + BaseHostVirtAddr: 0x400000, + Size: 0x3000, + Offset: 0x8000, + PageSize: 0x1000, + }, + } + + got, err := pageFaultCopyArgsForGuestOffset(regions, 0x9000, 7) + if err != nil { + t.Fatalf("pageFaultCopyArgsForGuestOffset returned error: %v", err) + } + + want := pageFaultCopyArgs{ + srcOffset: 0x9000, + dstAddr: 0x401000, + copyLen: 0x1000, + copyMode: 7, + } + if got != want { + t.Fatalf("pageFaultCopyArgsForGuestOffset() = %+v, want %+v", got, want) + } +} + +func TestPageFaultCopyArgsForGuestOffsetOutsideAllRegions(t *testing.T) { + regions := []GuestRegionUffdMapping{{ + BaseHostVirtAddr: 0x100000, + Size: 0x2000, + PageSize: 0x1000, + }} + + _, err := pageFaultCopyArgsForGuestOffset(regions, 0x3000, 0) + if !errors.Is(err, errGuestRegionNotFound) { + t.Fatalf("pageFaultCopyArgsForGuestOffset() error = %v, want %v", err, errGuestRegionNotFound) + } +} + +func TestTraceProcessRecordWritesWorkingSet(t *testing.T) { + baseDir := t.TempDir() + guestMemPath := filepath.Join(baseDir, "guest_mem") + workingSetPath := filepath.Join(baseDir, "working_set_pages") + pageSize := uint64(os.Getpagesize()) + + prepareGuestMemoryFile(t, guestMemPath, 5*int(pageSize)) + + trace := initTrace(filepath.Join(baseDir, "trace")) + trace.AppendRecord(Record{offset: 3 * pageSize}) + trace.AppendRecord(Record{offset: pageSize}) + trace.AppendRecord(Record{offset: 2 * pageSize}) + trace.AppendRecord(Record{offset: 2 * pageSize}) + + if err := trace.ProcessRecord(guestMemPath, workingSetPath, pageSize); err != nil { + t.Fatalf("ProcessRecord returned error: %v", err) + } + + if got, want := len(trace.trace), 3; got != want { + t.Fatalf("trace length = %d, want %d", got, want) + } + if got, want := trace.regions[pageSize], 3; got != want { + t.Fatalf("trace.regions[%#x] = %d, want %d", pageSize, got, want) + } + + got, err := os.ReadFile(workingSetPath) + if err != nil { + t.Fatalf("os.ReadFile working set returned error: %v", err) + } + wantGuest, err := os.ReadFile(guestMemPath) + if err != nil { + t.Fatalf("os.ReadFile guest memory returned error: %v", err) + } + want := wantGuest[pageSize : 4*pageSize] + if !reflect.DeepEqual(got, want) { + t.Fatal("working set contents do not match recorded guest memory pages") + } +} + func TestReceiveUffdMappingsAndFD(t *testing.T) { mappings := []GuestRegionUffdMapping{{ BaseHostVirtAddr: 0x100000, diff --git a/memory/manager/trace.go b/memory/manager/trace.go new file mode 100644 index 000000000..1bfe3f0a8 --- /dev/null +++ b/memory/manager/trace.go @@ -0,0 +1,210 @@ +// MIT License +// +// Copyright (c) 2020 Dmitrii Ustiugov, Plamen Petrov and EASE lab +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package manager + +import ( + "encoding/csv" + "errors" + "fmt" + "io" + "os" + "sort" + "strconv" + "sync" +) + +// Record identifies one guest memory page by its offset in the full memory file. +type Record struct { + offset uint64 +} + +// Trace stores recorded guest memory page offsets and replay regions. +type Trace struct { + sync.Mutex + traceFileName string + + pageSize uint64 + containedOffsets map[uint64]struct{} + trace []Record + regions map[uint64]int +} + +func initTrace(traceFileName string) *Trace { + return &Trace{ + traceFileName: traceFileName, + containedOffsets: make(map[uint64]struct{}), + trace: make([]Record, 0), + regions: make(map[uint64]int), + } +} + +func (t *Trace) AppendRecord(r Record) { + t.Lock() + defer t.Unlock() + + if _, ok := t.containedOffsets[r.offset]; ok { + return + } + t.trace = append(t.trace, r) + t.containedOffsets[r.offset] = struct{}{} +} + +func (t *Trace) WriteTrace() error { + t.Lock() + defer t.Unlock() + + file, err := os.Create(t.traceFileName) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + + writer := csv.NewWriter(file) + for _, rec := range t.trace { + if err := writer.Write([]string{strconv.FormatUint(rec.offset, 16)}); err != nil { + return err + } + } + writer.Flush() + return writer.Error() +} + +//nolint:unused +func (t *Trace) readTrace() error { + f, err := os.Open(t.traceFileName) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + lines, err := csv.NewReader(f).ReadAll() + if err != nil { + return err + } + + for _, line := range lines { + rec, err := readRecord(line) + if err != nil { + return err + } + t.AppendRecord(rec) + } + return nil +} + +//nolint:unused +func readRecord(line []string) (Record, error) { + if len(line) == 0 { + return Record{}, errors.New("empty trace record") + } + offset, err := strconv.ParseUint(line[0], 16, 64) + if err != nil { + return Record{}, err + } + + return Record{offset: offset}, nil +} + +func (t *Trace) containsRecord(rec Record) bool { + _, ok := t.containedOffsets[rec.offset] + return ok +} + +func (t *Trace) ProcessRecord(guestMemPath, workingSetPath string, pageSize uint64) error { + if pageSize == 0 { + return errInvalidGuestRegionPageSize + } + + t.Lock() + defer t.Unlock() + + t.pageSize = pageSize + sort.Slice(t.trace, func(i, j int) bool { + return t.trace[i].offset < t.trace[j].offset + }) + + t.regions = make(map[uint64]int) + var ( + last uint64 + regionStart uint64 + ) + for i, rec := range t.trace { + if i == 0 || rec.offset != last+pageSize { + regionStart = rec.offset + t.regions[regionStart] = 1 + } else { + t.regions[regionStart]++ + } + last = rec.offset + } + + return t.writeWorkingSetPagesToFileLocked(guestMemPath, workingSetPath, pageSize) +} + +func (t *Trace) writeWorkingSetPagesToFileLocked(guestMemPath, workingSetPath string, pageSize uint64) error { + fSrc, err := os.Open(guestMemPath) + if err != nil { + return err + } + defer func() { _ = fSrc.Close() }() + + fDst, err := os.Create(workingSetPath) + if err != nil { + return err + } + defer func() { _ = fDst.Close() }() + + keys := make([]uint64, 0, len(t.regions)) + for k := range t.regions { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + + var dstOffset int64 + for _, offset := range keys { + copyLen := uint64(t.regions[offset]) * pageSize + if copyLen > uint64(int(^uint(0)>>1)) { + return fmt.Errorf("working set region too large: %#x", copyLen) + } + + buf := make([]byte, int(copyLen)) + n, err := fSrc.ReadAt(buf, int64(offset)) + if err != nil && err != io.EOF { + return err + } + if n != len(buf) { + return io.ErrUnexpectedEOF + } + + n, err = fDst.WriteAt(buf, dstOffset) + if err != nil { + return err + } + if n != len(buf) { + return io.ErrShortWrite + } + dstOffset += int64(copyLen) + } + + return fDst.Sync() +} diff --git a/memory/manager/user_page_faults.h b/memory/manager/user_page_faults.h index a9f80b056..79fa0a833 100644 --- a/memory/manager/user_page_faults.h +++ b/memory/manager/user_page_faults.h @@ -26,5 +26,7 @@ #include // constants for use from Go +int const_UFFDIO_WAKE = UFFDIO_WAKE; int const_UFFDIO_COPY = UFFDIO_COPY; int const_UFFD_EVENT_PAGEFAULT = UFFD_EVENT_PAGEFAULT; +int const_UFFDIO_COPY_MODE_DONTWAKE = UFFDIO_COPY_MODE_DONTWAKE; diff --git a/vhive.go b/vhive.go index c1bddf843..be784825e 100644 --- a/vhive.go +++ b/vhive.go @@ -97,6 +97,10 @@ func main() { log.Error("User-level page faults are not supported without snapshots") return } + if *isUPFEnabled && !*isLazyMode { + log.Error("User-level page faults currently require lazy serving mode") + return + } if !*isUPFEnabled && *isLazyMode { log.Error("Lazy page fault serving mode is not supported without user-level page faults") diff --git a/vhive_test.go b/vhive_test.go index 9c5450db9..d663ee294 100644 --- a/vhive_test.go +++ b/vhive_test.go @@ -64,6 +64,10 @@ func TestMain(m *testing.M) { log.SetLevel(log.InfoLevel) flag.Parse() + if *isUPFEnabledTest && !*isLazyModeTest { + log.Error("User-level page faults currently require lazy serving mode") + os.Exit(-1) + } log.Infof("Orchestrator snapshots enabled: %t", *isSnapshotsEnabledTest) log.Infof("Orchestrator UPF enabled: %t", *isUPFEnabledTest) From 3e882fa1448415264703cf559142e035f9034853 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Mon, 13 Jul 2026 18:39:04 +0800 Subject: [PATCH 17/21] comments follow-up Signed-off-by: Liquidzk <492928615@qq.com> --- ctriface/iface.go | 35 ++++++++++++++-------------------- memory/manager/manager.go | 23 +++++++++------------- memory/manager/manager_test.go | 28 ++++++++++++++++++++++----- 3 files changed, 46 insertions(+), 40 deletions(-) diff --git a/ctriface/iface.go b/ctriface/iface.go index 1933c6b64..efeb75493 100644 --- a/ctriface/iface.go +++ b/ctriface/iface.go @@ -525,7 +525,7 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap loadSnapshotMetric = metrics.NewMetric() tStart time.Time loadErr, activateErr error - loadDone = make(chan int) + activateDone chan error ) logger := log.WithFields(log.Fields{"vmID": vmID}) @@ -633,34 +633,27 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap tStart = time.Now() if o.GetUPFEnabled() { - activateDone := make(chan error, 1) + activateDone = make(chan error, 1) socketReady := make(chan error, 1) go func() { - activateDone <- o.memoryManager.ActivateWithSocketReady(vmID, socketReady) + err := o.memoryManager.Activate(vmID, socketReady) + if err != nil { + logger.WithError(err).Warn("Failed to activate VM in the memory manager") + } + activateDone <- err }() if err := <-socketReady; err != nil { - activateErr = <-activateDone - return nil, nil, multierror.Of(err, activateErr) - } - - if _, loadErr = o.fcClient.CreateVM(ctx, conf); loadErr != nil { - logSnapshotLoadFailure(logger, snap, conf, loadErr) + return nil, nil, multierror.Of(err, <-activateDone) } - activateErr = <-activateDone - if activateErr != nil { - logger.Warn("Failed to activate VM in the memory manager", activateErr) - } - } else { - go func() { - defer close(loadDone) + } - if _, loadErr = o.fcClient.CreateVM(ctx, conf); loadErr != nil { - logSnapshotLoadFailure(logger, snap, conf, loadErr) - } - }() + if _, loadErr = o.fcClient.CreateVM(ctx, conf); loadErr != nil { + logSnapshotLoadFailure(logger, snap, conf, loadErr) + } - <-loadDone + if activateDone != nil { + activateErr = <-activateDone } loadSnapshotMetric.MetricMap[metrics.LoadVMM] = metrics.ToUS(time.Since(tStart)) diff --git a/memory/manager/manager.go b/memory/manager/manager.go index d007eed9f..8a5be165e 100644 --- a/memory/manager/manager.go +++ b/memory/manager/manager.go @@ -145,18 +145,9 @@ func (m *MemoryManager) DeregisterVM(vmID string) error { return nil } -// Activate Creates an epoller to serve page faults for the VM -func (m *MemoryManager) Activate(vmID string) error { - return m.activate(vmID, nil) -} - -// ActivateWithSocketReady is like Activate, but reports when the UFFD socket -// listener is ready for Firecracker to connect. -func (m *MemoryManager) ActivateWithSocketReady(vmID string, socketReadyCh chan<- error) error { - return m.activate(vmID, socketReadyCh) -} - -func (m *MemoryManager) activate(vmID string, socketReadyCh chan<- error) error { +// Activate creates an epoller to serve page faults and reports when the UFFD +// socket listener is ready for Firecracker to connect. +func (m *MemoryManager) Activate(vmID string, socketReadyCh chan<- error) error { logger := log.WithFields(log.Fields{"vmID": vmID}) logger.Debug("Activating instance in the memory manager") @@ -173,14 +164,18 @@ func (m *MemoryManager) activate(vmID string, socketReadyCh chan<- error) error if !ok { m.Unlock() logger.Error("VM not registered with the memory manager") - return errors.New("VM not registered with the memory manager") + err := errors.New("VM not registered with the memory manager") + notifySocketReady(socketReadyCh, err) + return err } m.Unlock() if state.isActive { logger.Error("VM already active") - return errors.New("VM already active") + err := errors.New("VM already active") + notifySocketReady(socketReadyCh, err) + return err } if err := state.mapGuestMemory(); err != nil { diff --git a/memory/manager/manager_test.go b/memory/manager/manager_test.go index 4ac83b2b3..e39b9277b 100644 --- a/memory/manager/manager_test.go +++ b/memory/manager/manager_test.go @@ -196,11 +196,11 @@ func TestMemoryManagerActivateReceivesFirecrackerMappings(t *testing.T) { socketReadyCh := make(chan error, 1) activateErrCh := make(chan error, 1) go func() { - activateErrCh <- manager.ActivateWithSocketReady(vmID, socketReadyCh) + activateErrCh <- manager.Activate(vmID, socketReadyCh) }() if err := receiveSocketReady(t, socketReadyCh); err != nil { - t.Fatalf("ActivateWithSocketReady failed before socket accept: %v", err) + t.Fatalf("Activate failed before socket accept: %v", err) } conn := dialUnixSocketWithRetry(t, socketPath) @@ -213,12 +213,12 @@ func TestMemoryManagerActivateReceivesFirecrackerMappings(t *testing.T) { } if err := receiveActivateResult(t, activateErrCh); err != nil { - t.Fatalf("ActivateWithSocketReady returned error: %v", err) + t.Fatalf("Activate returned error: %v", err) } state := manager.instances[vmID] if !state.isActive { - t.Fatal("state is not active after ActivateWithSocketReady") + t.Fatal("state is not active after Activate") } if !reflect.DeepEqual(state.guestRegionMappings, mappings) { t.Fatalf("guestRegionMappings = %+v, want %+v", state.guestRegionMappings, mappings) @@ -235,6 +235,24 @@ func TestMemoryManagerActivateReceivesFirecrackerMappings(t *testing.T) { } } +func TestMemoryManagerActivateReportsErrorBeforeSocketListen(t *testing.T) { + manager := NewMemoryManager(MemoryManagerCfg{}) + socketReadyCh := make(chan error, 1) + + activateErr := manager.Activate("missing-vm", socketReadyCh) + if activateErr == nil { + t.Fatal("Activate returned nil error for an unregistered VM") + } + + readyErr := receiveSocketReady(t, socketReadyCh) + if readyErr == nil { + t.Fatal("socket readiness returned nil error for an unregistered VM") + } + if readyErr.Error() != activateErr.Error() { + t.Fatalf("socket readiness error = %q, want %q", readyErr, activateErr) + } +} + func receiveSocketReady(t *testing.T, readyCh <-chan error) error { t.Helper() @@ -255,7 +273,7 @@ func receiveActivateResult(t *testing.T, activateErrCh <-chan error) error { case err := <-activateErrCh: return err case <-time.After(time.Second): - t.Fatal("timed out waiting for ActivateWithSocketReady") + t.Fatal("timed out waiting for Activate") } return nil From 43f474fbad5d1c0f882a3f4f86db4e553e0d43d9 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Tue, 14 Jul 2026 11:11:01 +0800 Subject: [PATCH 18/21] Update firecracker-containerd binaries Signed-off-by: Liquidzk <492928615@qq.com> --- bin/containerd-shim-aws-firecracker | 4 ++-- bin/firecracker-containerd | 4 ++-- bin/firecracker-ctr | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/containerd-shim-aws-firecracker b/bin/containerd-shim-aws-firecracker index e2b219c37..6eb06eec5 100755 --- a/bin/containerd-shim-aws-firecracker +++ b/bin/containerd-shim-aws-firecracker @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:81ff81fc3169fc4fafcd7e33588dc3ba528665b39f17cbe595e14dfd7690b118 -size 37271800 +oid sha256:e91d81910586b88ab86354d745630267c983de790df3c3ce0ddf2639bfa752f3 +size 41216680 diff --git a/bin/firecracker-containerd b/bin/firecracker-containerd index 7cb01c83b..39e2069e5 100755 --- a/bin/firecracker-containerd +++ b/bin/firecracker-containerd @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1959d901c4a8a6bdf8394628d5c62c0d6eba23a976acfdf8f9173fc21bc26e68 -size 69041344 +oid sha256:26e2b09dc4d8026b4bdcf1fc7258cc12ad1b9cbf3f43fb79a032281d9c31b287 +size 83184416 diff --git a/bin/firecracker-ctr b/bin/firecracker-ctr index 40d64e16f..691e76aff 100755 --- a/bin/firecracker-ctr +++ b/bin/firecracker-ctr @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bfc28b8f8092d10190a4f7fc74cea36be12a9ef9f4ec81a3b7b46cde7d7ea857 -size 33034096 +oid sha256:030da5847deaa90c25a124634d5969698ecd89f68094d0700e69433410d32948 +size 42211328 From 4a060b1bc7540f8c398a135969f040189f182e02 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Tue, 14 Jul 2026 15:59:23 +0800 Subject: [PATCH 19/21] Address UPF review follow-up Signed-off-by: Liquidzk <492928615@qq.com> --- .github/workflows/unit_tests.yml | 8 ++------ ctriface/iface.go | 8 +++++--- ctriface/iface_test.go | 10 ---------- memory/manager/Makefile | 33 ++++++++++++++++++++++++++++++++ memory/manager/manager.go | 11 +++-------- memory/manager/manager_test.go | 28 +++++++++++---------------- memory/manager/snapshot_state.go | 15 +++++---------- 7 files changed, 59 insertions(+), 54 deletions(-) create mode 100644 memory/manager/Makefile diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 51bbf8bd7..731c7e4e8 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -64,12 +64,8 @@ jobs: AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} run: | - if [[ "$MODULE" == "memory/manager" ]]; then - go test -race ./memory/manager -count=1 - else - make -C "$MODULE" test - make -C "$MODULE" test-man - fi + make -C "$MODULE" test + make -C "$MODULE" test-man profile-unit-test: if: false diff --git a/ctriface/iface.go b/ctriface/iface.go index efeb75493..fe8054e6a 100644 --- a/ctriface/iface.go +++ b/ctriface/iface.go @@ -634,7 +634,7 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap if o.GetUPFEnabled() { activateDone = make(chan error, 1) - socketReady := make(chan error, 1) + socketReady := make(chan struct{}, 1) go func() { err := o.memoryManager.Activate(vmID, socketReady) if err != nil { @@ -643,8 +643,10 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap activateDone <- err }() - if err := <-socketReady; err != nil { - return nil, nil, multierror.Of(err, <-activateDone) + select { + case <-socketReady: + case activateErr = <-activateDone: + return nil, nil, activateErr } } diff --git a/ctriface/iface_test.go b/ctriface/iface_test.go index 52d30a26c..6a7b0eb32 100644 --- a/ctriface/iface_test.go +++ b/ctriface/iface_test.go @@ -54,16 +54,6 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestValidateUPFModeRequiresLazyMode(t *testing.T) { - orch := &Orchestrator{} - WithUPF(true)(orch) - - require.ErrorIs(t, orch.validateUPFMode(), errUPFRequiresLazyMode) - - WithLazyMode(true)(orch) - require.NoError(t, orch.validateUPFMode()) -} - func TestStartSnapStopLoad(t *testing.T) { log.SetFormatter(&log.TextFormatter{ TimestampFormat: ctrdlog.RFC3339NanoFixed, diff --git a/memory/manager/Makefile b/memory/manager/Makefile new file mode 100644 index 000000000..6d60adc8c --- /dev/null +++ b/memory/manager/Makefile @@ -0,0 +1,33 @@ +# MIT License +# +# Copyright (c) 2023 Georgiy Lebedev, Dmitrii Ustiugov, Plamen Petrov and vHive team +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +EXTRAGOARGS:=-v -race -cover + +test: + # Need to pass GOROOT because GitHub-hosted runners may have several + # go versions installed so that calling go from root may fail + sudo env "PATH=$(PATH)" "GOROOT=$(GOROOT)" go test ./ $(EXTRAGOARGS) + +test-man: + echo "Nothing to test manually" + +.PHONY: test test-man diff --git a/memory/manager/manager.go b/memory/manager/manager.go index 8a5be165e..e25ab5c1a 100644 --- a/memory/manager/manager.go +++ b/memory/manager/manager.go @@ -147,7 +147,7 @@ func (m *MemoryManager) DeregisterVM(vmID string) error { // Activate creates an epoller to serve page faults and reports when the UFFD // socket listener is ready for Firecracker to connect. -func (m *MemoryManager) Activate(vmID string, socketReadyCh chan<- error) error { +func (m *MemoryManager) Activate(vmID string, socketReadyCh chan<- struct{}) error { logger := log.WithFields(log.Fields{"vmID": vmID}) logger.Debug("Activating instance in the memory manager") @@ -164,23 +164,18 @@ func (m *MemoryManager) Activate(vmID string, socketReadyCh chan<- error) error if !ok { m.Unlock() logger.Error("VM not registered with the memory manager") - err := errors.New("VM not registered with the memory manager") - notifySocketReady(socketReadyCh, err) - return err + return errors.New("VM not registered with the memory manager") } m.Unlock() if state.isActive { logger.Error("VM already active") - err := errors.New("VM already active") - notifySocketReady(socketReadyCh, err) - return err + return errors.New("VM already active") } if err := state.mapGuestMemory(); err != nil { logger.Error("Failed to map guest memory") - notifySocketReady(socketReadyCh, err) return err } diff --git a/memory/manager/manager_test.go b/memory/manager/manager_test.go index e39b9277b..04b0e871d 100644 --- a/memory/manager/manager_test.go +++ b/memory/manager/manager_test.go @@ -193,15 +193,13 @@ func TestMemoryManagerActivateReceivesFirecrackerMappings(t *testing.T) { } uffdStandIn := testEventFD(t) - socketReadyCh := make(chan error, 1) + socketReadyCh := make(chan struct{}, 1) activateErrCh := make(chan error, 1) go func() { activateErrCh <- manager.Activate(vmID, socketReadyCh) }() - if err := receiveSocketReady(t, socketReadyCh); err != nil { - t.Fatalf("Activate failed before socket accept: %v", err) - } + receiveSocketReady(t, socketReadyCh) conn := dialUnixSocketWithRetry(t, socketPath) if err := writeUffdSocketPayload(conn, body, uffdStandIn); err != nil { @@ -235,35 +233,31 @@ func TestMemoryManagerActivateReceivesFirecrackerMappings(t *testing.T) { } } -func TestMemoryManagerActivateReportsErrorBeforeSocketListen(t *testing.T) { +func TestMemoryManagerActivateReturnsErrorBeforeSocketListen(t *testing.T) { manager := NewMemoryManager(MemoryManagerCfg{}) - socketReadyCh := make(chan error, 1) + socketReadyCh := make(chan struct{}, 1) activateErr := manager.Activate("missing-vm", socketReadyCh) if activateErr == nil { t.Fatal("Activate returned nil error for an unregistered VM") } - readyErr := receiveSocketReady(t, socketReadyCh) - if readyErr == nil { - t.Fatal("socket readiness returned nil error for an unregistered VM") - } - if readyErr.Error() != activateErr.Error() { - t.Fatalf("socket readiness error = %q, want %q", readyErr, activateErr) + select { + case <-socketReadyCh: + t.Fatal("socket readiness was reported for an unregistered VM") + default: } } -func receiveSocketReady(t *testing.T, readyCh <-chan error) error { +func receiveSocketReady(t *testing.T, readyCh <-chan struct{}) { t.Helper() select { - case err := <-readyCh: - return err + case <-readyCh: + return case <-time.After(time.Second): t.Fatal("timed out waiting for UFFD socket readiness") } - - return nil } func receiveActivateResult(t *testing.T, activateErrCh <-chan error) error { diff --git a/memory/manager/snapshot_state.go b/memory/manager/snapshot_state.go index c8809d325..9755d9fe2 100644 --- a/memory/manager/snapshot_state.go +++ b/memory/manager/snapshot_state.go @@ -347,7 +347,7 @@ func (s *SnapshotState) setupStateOnActivate() { } } -func (s *SnapshotState) getUFFD(socketReadyCh chan<- error) error { +func (s *SnapshotState) getUFFD(socketReadyCh chan<- struct{}) error { mappings, userFaultFD, err := receiveUffdMappingsAndFDFromSocket(s.InstanceSockAddr, socketReadyCh) if err != nil { log.Error("Failed to receive the uffd and guest memory mappings") @@ -360,35 +360,30 @@ func (s *SnapshotState) getUFFD(socketReadyCh chan<- error) error { return nil } -func receiveUffdMappingsAndFDFromSocket(socketPath string, socketReadyCh chan<- error) ([]GuestRegionUffdMapping, *os.File, error) { +func receiveUffdMappingsAndFDFromSocket(socketPath string, socketReadyCh chan<- struct{}) ([]GuestRegionUffdMapping, *os.File, error) { if socketPath == "" { - notifySocketReady(socketReadyCh, errEmptyUffdSocketPath) return nil, nil, errEmptyUffdSocketPath } if err := os.MkdirAll(filepath.Dir(socketPath), 0755); err != nil { - notifySocketReady(socketReadyCh, err) return nil, nil, err } if err := removeStaleUffdSocket(socketPath); err != nil { - notifySocketReady(socketReadyCh, err) return nil, nil, err } addr := &net.UnixAddr{Name: socketPath, Net: "unix"} listener, err := net.ListenUnix("unix", addr) if err != nil { - notifySocketReady(socketReadyCh, err) return nil, nil, err } defer func() { _ = listener.Close() }() defer func() { _ = os.Remove(socketPath) }() if err := listener.SetDeadline(time.Now().Add(uffdSocketAcceptTimeout)); err != nil { - notifySocketReady(socketReadyCh, err) return nil, nil, err } - notifySocketReady(socketReadyCh, nil) + notifySocketReady(socketReadyCh) conn, err := listener.AcceptUnix() if err != nil { @@ -399,11 +394,11 @@ func receiveUffdMappingsAndFDFromSocket(socketPath string, socketReadyCh chan<- return receiveUffdMappingsAndFD(conn) } -func notifySocketReady(socketReadyCh chan<- error, err error) { +func notifySocketReady(socketReadyCh chan<- struct{}) { if socketReadyCh == nil { return } - socketReadyCh <- err + socketReadyCh <- struct{}{} } func receiveUffdMappingsAndFD(conn *net.UnixConn) ([]GuestRegionUffdMapping, *os.File, error) { From 3b6c9894372d3f68c463bfc5c228b460eee67067 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Tue, 14 Jul 2026 20:37:09 +0800 Subject: [PATCH 20/21] Enable UPF interface tests Signed-off-by: Liquidzk <492928615@qq.com> --- ctriface/Makefile | 21 +++++---------------- ctriface/iface_test.go | 2 ++ 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/ctriface/Makefile b/ctriface/Makefile index a23753c43..182b3a8a2 100644 --- a/ctriface/Makefile +++ b/ctriface/Makefile @@ -23,11 +23,7 @@ EXTRAGOARGS:=-v -race -cover EXTRATESTFILES:=iface_test.go iface.go orch_options.go orch.go BENCHFILES:=bench_test.go iface.go orch_options.go orch.go -# User-level page faults are temporarily disabled (gh-807) -# WITHUPF:=-upf -# WITHLAZY:=-lazy -WITHUPF:= -WITHLAZY:= +UPFARGS:=-upf -lazy STARGZ:=-ss 'proxy' -img 'ghcr.io/vhive-serverless/helloworld:var_workload-esgz' DOCKER_CREDENTIALS:=-dockerCredentials '{"docker-credentials":{"ghcr.io":{"username":"","password":""}}}' GOBENCH:=-v -timeout 1500s @@ -39,7 +35,7 @@ test: sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) ./../scripts/clean_fcctr.sh sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log.out 2>$(CTRDLOGDIR)/ctriface_log.err & - sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) -args $(WITHUPF) + sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) -args $(UPFARGS) ./../scripts/clean_fcctr.sh test-man: @@ -48,10 +44,7 @@ test-man: sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestSnapLoad ./../scripts/clean_fcctr.sh sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_upf_man_travis.out 2>$(CTRDLOGDIR)/ctriface_log_upf_man_travis.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestSnapLoad -args $(WITHUPF) - ./../scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_travis.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestSnapLoad -args $(WITHUPF) $(WITHLAZY) + sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestSnapLoad -args $(UPFARGS) ./../scripts/clean_fcctr.sh sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_remote_snap_create_man_travis.out 2>$(CTRDLOGDIR)/ctriface_log_remote_snap_create_man_travis.err & # Creates a remote snapshot. @@ -74,12 +67,8 @@ test-skip: sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelPhasedSnapLoad ./../scripts/clean_fcctr.sh sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_upf_man_skip.out 2>$(CTRDLOGDIR)/ctriface_log_upf_man_skip.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelSnapLoad -args $(WITHUPF) - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelPhasedSnapLoad -args $(WITHUPF) - ./../scripts/clean_fcctr.sh - sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_lazy_man_skip.out 2>$(CTRDLOGDIR)/ctriface_log_lazy_man_skip.err & - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelSnapLoad -args $(WITHUPF) $(WITHLAZY) - sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelPhasedSnapLoad -args $(WITHUPF) $(WITHLAZY) + sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelSnapLoad -args $(UPFARGS) + sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestParallelPhasedSnapLoad -args $(UPFARGS) ./../scripts/clean_fcctr.sh diff --git a/ctriface/iface_test.go b/ctriface/iface_test.go index 6a7b0eb32..41be55bec 100644 --- a/ctriface/iface_test.go +++ b/ctriface/iface_test.go @@ -73,6 +73,8 @@ func TestStartSnapStopLoad(t *testing.T) { *snapshotter, "", WithTestModeOn(true), + WithUPF(*isUPFEnabled), + WithLazyMode(*isLazyMode), WithDockerCredentials(*dockerCredentials), ) From e3711129d869cd6e78b369719849254414527c46 Mon Sep 17 00:00:00 2001 From: Liquidzk <492928615@qq.com> Date: Thu, 16 Jul 2026 21:15:10 +0800 Subject: [PATCH 21/21] code structure follow-up Signed-off-by: Liquidzk <492928615@qq.com> --- ctriface/iface.go | 37 +- go.mod | 4 +- go.sum | 4 +- memory/manager/snapshot_state.go | 701 +---------------------------- memory/manager/uffd.go | 741 +++++++++++++++++++++++++++++++ 5 files changed, 769 insertions(+), 718 deletions(-) create mode 100644 memory/manager/uffd.go diff --git a/ctriface/iface.go b/ctriface/iface.go index fe8054e6a..1d5979aa0 100644 --- a/ctriface/iface.go +++ b/ctriface/iface.go @@ -522,10 +522,12 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap } var ( - loadSnapshotMetric = metrics.NewMetric() - tStart time.Time - loadErr, activateErr error - activateDone chan error + loadSnapshotMetric = metrics.NewMetric() + tStart time.Time + loadErr error + activateErr error + deactivateErr error + activateErrChan chan error ) logger := log.WithFields(log.Fields{"vmID": vmID}) @@ -633,19 +635,19 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap tStart = time.Now() if o.GetUPFEnabled() { - activateDone = make(chan error, 1) - socketReady := make(chan struct{}, 1) + activateErrChan = make(chan error, 1) + socketReadyChan := make(chan struct{}, 1) go func() { - err := o.memoryManager.Activate(vmID, socketReady) + err := o.memoryManager.Activate(vmID, socketReadyChan) if err != nil { logger.WithError(err).Warn("Failed to activate VM in the memory manager") } - activateDone <- err + activateErrChan <- err }() select { - case <-socketReady: - case activateErr = <-activateDone: + case <-socketReadyChan: + case activateErr = <-activateErrChan: return nil, nil, activateErr } } @@ -654,14 +656,21 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap logSnapshotLoadFailure(logger, snap, conf, loadErr) } - if activateDone != nil { - activateErr = <-activateDone + if activateErrChan != nil { + activateErr = <-activateErrChan + } + + if loadErr != nil && activateErr == nil && activateErrChan != nil { + deactivateErr = o.memoryManager.Deactivate(vmID) + if deactivateErr != nil { + logger.WithError(deactivateErr).Warn("Failed to deactivate VM in the memory manager after snapshot load failure") + } } loadSnapshotMetric.MetricMap[metrics.LoadVMM] = metrics.ToUS(time.Since(tStart)) - if loadErr != nil || activateErr != nil { - multierr := multierror.Of(loadErr, activateErr) + if loadErr != nil || activateErr != nil || deactivateErr != nil { + multierr := multierror.Of(loadErr, activateErr, deactivateErr) return nil, nil, multierr } diff --git a/go.mod b/go.mod index 625490ef7..7026e0b20 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/vhive-serverless/vhive go 1.26.2 replace ( - github.com/firecracker-microvm/firecracker-containerd => github.com/vhive-serverless/firecracker-containerd v0.0.0-20260205055657-a35b52c279d9 + github.com/firecracker-microvm/firecracker-containerd => github.com/vhive-serverless/firecracker-containerd v0.0.0-20260713114451-bb007a1ef569 github.com/vhive-serverless/vhive/examples/protobuf/helloworld => ./examples/protobuf/helloworld ) @@ -31,6 +31,7 @@ require ( gonum.org/v1/gonum v0.17.0 gonum.org/v1/plot v0.17.0 google.golang.org/grpc v1.82.0 + google.golang.org/protobuf v1.36.11 k8s.io/cri-api v0.27.1 ) @@ -90,6 +91,5 @@ require ( golang.org/x/text v0.37.0 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ff7641417..c585e9c94 100644 --- a/go.sum +++ b/go.sum @@ -910,8 +910,8 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= -github.com/vhive-serverless/firecracker-containerd v0.0.0-20260205055657-a35b52c279d9 h1:Q2ybUII/hVdXKMA35KX54VLsrmydZ/HxxIKUxI6L31M= -github.com/vhive-serverless/firecracker-containerd v0.0.0-20260205055657-a35b52c279d9/go.mod h1:JDguJbk1e92EIfqyO7VRIeljZy3vw+JsJO62JGsl2i8= +github.com/vhive-serverless/firecracker-containerd v0.0.0-20260713114451-bb007a1ef569 h1:I5OIQ1m6i11vxp85C12CudEBGojDjc4jqQr0E8j8AKs= +github.com/vhive-serverless/firecracker-containerd v0.0.0-20260713114451-bb007a1ef569/go.mod h1:IlWnPBeebJFzQ0G12229J/hy9f4RhfdDTRC2+qAaXh0= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= diff --git a/memory/manager/snapshot_state.go b/memory/manager/snapshot_state.go index 9755d9fe2..f7f54eea3 100644 --- a/memory/manager/snapshot_state.go +++ b/memory/manager/snapshot_state.go @@ -22,187 +22,19 @@ package manager -/* -#include "user_page_faults.h" -*/ -import "C" - import ( - "encoding/binary" - "encoding/json" - "errors" "fmt" "io" - "math" - "net" "os" "path/filepath" - "sort" "sync" - "syscall" - "time" log "github.com/sirupsen/logrus" "golang.org/x/sys/unix" "github.com/vhive-serverless/vhive/metrics" - - "unsafe" -) - -const ( - uffdSocketPayloadSize = 64 * 1024 - uffdSocketFDLimit = 2 - uffdSocketReadTimeout = time.Second - uffdSocketAcceptTimeout = 30 * time.Second ) -var ( - errInvalidGuestRegionPageSize = errors.New("guest region page size must be non-zero") - errGuestRegionNotFound = errors.New("fault address is outside guest memory mappings") - errUnexpectedUffdFDCount = errors.New("expected exactly one uffd fd") - errNoGuestRegionMappings = errors.New("no guest region mappings received") - errEmptyUffdSocketPath = errors.New("empty uffd socket path") -) - -// GuestRegionUffdMapping describes Firecracker's UFFD guest memory mapping. -type GuestRegionUffdMapping struct { - BaseHostVirtAddr uint64 `json:"base_host_virt_addr"` - Size uint64 `json:"size"` - Offset uint64 `json:"offset"` - PageSize uint64 `json:"page_size"` -} - -type pageFaultCopyArgs struct { - srcOffset uint64 - dstAddr uint64 - copyLen uint64 - copyMode uint64 -} - -func pageAlignFaultAddress(faultAddr uint64, region GuestRegionUffdMapping) (uint64, error) { - if region.PageSize == 0 { - return 0, errInvalidGuestRegionPageSize - } - - return faultAddr - faultAddr%region.PageSize, nil -} - -func guestMemoryOffsetForFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) (uint64, error) { - if region.PageSize == 0 { - return 0, errInvalidGuestRegionPageSize - } - if !regionContainsFaultPage(region, faultPageAddr) { - return 0, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultPageAddr) - } - - regionOffset := faultPageAddr - region.BaseHostVirtAddr - if region.Offset > math.MaxUint64-regionOffset { - return 0, fmt.Errorf("guest memory offset overflow for fault address %#x", faultPageAddr) - } - - return region.Offset + regionOffset, nil -} - -func guestAddressForMemoryOffset(region GuestRegionUffdMapping, offset uint64) (uint64, error) { - if region.PageSize == 0 { - return 0, errInvalidGuestRegionPageSize - } - if !regionContainsGuestMemoryOffset(region, offset) { - return 0, fmt.Errorf("%w: %#x", errGuestRegionNotFound, offset) - } - - regionOffset := offset - region.Offset - if region.BaseHostVirtAddr > math.MaxUint64-regionOffset { - return 0, fmt.Errorf("guest address overflow for memory offset %#x", offset) - } - - return region.BaseHostVirtAddr + regionOffset, nil -} - -func pageFaultCopyArgsForFault(regions []GuestRegionUffdMapping, faultAddr uint64) (pageFaultCopyArgs, error) { - for _, region := range regions { - faultPageAddr, err := pageAlignFaultAddress(faultAddr, region) - if err != nil { - return pageFaultCopyArgs{}, err - } - if !regionContainsFaultPage(region, faultPageAddr) { - continue - } - - srcOffset, err := guestMemoryOffsetForFaultPage(region, faultPageAddr) - if err != nil { - return pageFaultCopyArgs{}, err - } - - return pageFaultCopyArgs{ - srcOffset: srcOffset, - dstAddr: faultPageAddr, - copyLen: region.PageSize, - copyMode: 0, - }, nil - } - - return pageFaultCopyArgs{}, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultAddr) -} - -func pageFaultCopyArgsForGuestOffset(regions []GuestRegionUffdMapping, offset uint64, mode uint64) (pageFaultCopyArgs, error) { - for _, region := range regions { - if !regionContainsGuestMemoryOffset(region, offset) { - continue - } - dstAddr, err := guestAddressForMemoryOffset(region, offset) - if err != nil { - return pageFaultCopyArgs{}, err - } - - return pageFaultCopyArgs{ - srcOffset: offset, - dstAddr: dstAddr, - copyLen: region.PageSize, - copyMode: mode, - }, nil - } - - return pageFaultCopyArgs{}, fmt.Errorf("%w: %#x", errGuestRegionNotFound, offset) -} - -func regionContainsFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) bool { - if region.Size == 0 || faultPageAddr < region.BaseHostVirtAddr { - return false - } - - return faultPageAddr-region.BaseHostVirtAddr < region.Size -} - -func regionContainsGuestMemoryOffset(region GuestRegionUffdMapping, offset uint64) bool { - if region.Size == 0 || offset < region.Offset { - return false - } - - return offset-region.Offset < region.Size -} - -func guestMappingPageSize(regions []GuestRegionUffdMapping) (uint64, error) { - var pageSize uint64 - for _, region := range regions { - if region.PageSize == 0 { - return 0, errInvalidGuestRegionPageSize - } - if pageSize == 0 { - pageSize = region.PageSize - continue - } - if pageSize != region.PageSize { - return 0, errors.New("mixed guest region page sizes are not supported for working-set replay") - } - } - if pageSize == 0 { - return 0, errNoGuestRegionMappings - } - return pageSize, nil -} - // SnapshotStateCfg Config to initialize SnapshotState type SnapshotStateCfg struct { VMID string @@ -347,146 +179,6 @@ func (s *SnapshotState) setupStateOnActivate() { } } -func (s *SnapshotState) getUFFD(socketReadyCh chan<- struct{}) error { - mappings, userFaultFD, err := receiveUffdMappingsAndFDFromSocket(s.InstanceSockAddr, socketReadyCh) - if err != nil { - log.Error("Failed to receive the uffd and guest memory mappings") - return err - } - - s.guestRegionMappings = mappings - s.userFaultFD = userFaultFD - - return nil -} - -func receiveUffdMappingsAndFDFromSocket(socketPath string, socketReadyCh chan<- struct{}) ([]GuestRegionUffdMapping, *os.File, error) { - if socketPath == "" { - return nil, nil, errEmptyUffdSocketPath - } - if err := os.MkdirAll(filepath.Dir(socketPath), 0755); err != nil { - return nil, nil, err - } - if err := removeStaleUffdSocket(socketPath); err != nil { - return nil, nil, err - } - - addr := &net.UnixAddr{Name: socketPath, Net: "unix"} - listener, err := net.ListenUnix("unix", addr) - if err != nil { - return nil, nil, err - } - defer func() { _ = listener.Close() }() - defer func() { _ = os.Remove(socketPath) }() - - if err := listener.SetDeadline(time.Now().Add(uffdSocketAcceptTimeout)); err != nil { - return nil, nil, err - } - - notifySocketReady(socketReadyCh) - - conn, err := listener.AcceptUnix() - if err != nil { - return nil, nil, err - } - defer func() { _ = conn.Close() }() - - return receiveUffdMappingsAndFD(conn) -} - -func notifySocketReady(socketReadyCh chan<- struct{}) { - if socketReadyCh == nil { - return - } - socketReadyCh <- struct{}{} -} - -func receiveUffdMappingsAndFD(conn *net.UnixConn) ([]GuestRegionUffdMapping, *os.File, error) { - if err := conn.SetReadDeadline(time.Now().Add(uffdSocketReadTimeout)); err != nil { - return nil, nil, err - } - defer func() { _ = conn.SetReadDeadline(time.Time{}) }() - - body := make([]byte, uffdSocketPayloadSize) - oob := make([]byte, unix.CmsgSpace(uffdSocketFDLimit*4)) - - n, oobn, flags, _, err := conn.ReadMsgUnix(body, oob) - if err != nil { - return nil, nil, err - } - if flags&unix.MSG_TRUNC != 0 { - return nil, nil, errors.New("uffd mappings payload was truncated") - } - if flags&unix.MSG_CTRUNC != 0 { - return nil, nil, errors.New("uffd fd control message was truncated") - } - - fds, err := parseUnixRights(oob[:oobn]) - if err != nil { - return nil, nil, err - } - if len(fds) != 1 { - closeFDs(fds) - return nil, nil, fmt.Errorf("%w: got %d", errUnexpectedUffdFDCount, len(fds)) - } - - uffdFile := os.NewFile(uintptr(fds[0]), "userfaultfd") - if uffdFile == nil { - return nil, nil, errors.New("failed to create file for uffd fd") - } - - var mappings []GuestRegionUffdMapping - if err := json.Unmarshal(body[:n], &mappings); err != nil { - _ = uffdFile.Close() - return nil, nil, fmt.Errorf("cannot deserialize memory mappings: %w", err) - } - if len(mappings) == 0 { - _ = uffdFile.Close() - return nil, nil, errNoGuestRegionMappings - } - - return mappings, uffdFile, nil -} - -func parseUnixRights(oob []byte) ([]int, error) { - scms, err := unix.ParseSocketControlMessage(oob) - if err != nil { - return nil, err - } - - var fds []int - for i := range scms { - rights, err := unix.ParseUnixRights(&scms[i]) - if err != nil { - closeFDs(fds) - return nil, err - } - fds = append(fds, rights...) - } - - return fds, nil -} - -func closeFDs(fds []int) { - for _, receivedFD := range fds { - _ = unix.Close(receivedFD) - } -} - -func removeStaleUffdSocket(socketPath string) error { - info, err := os.Lstat(socketPath) - if err == nil { - if info.Mode()&os.ModeSocket == 0 { - return fmt.Errorf("refusing to remove non-socket uffd path %q", socketPath) - } - return os.Remove(socketPath) - } - if os.IsNotExist(err) { - return nil - } - return err -} - func (s *SnapshotState) processMetrics() { if !s.metricsModeOn || s.currentMetric == nil { return @@ -570,7 +262,7 @@ func (s *SnapshotState) fetchWorkingSet() error { } defer func() { _ = f.Close() }() - s.workingSet = AlignedBlock(int(size)) + s.workingSet = make([]byte, int(size)) n, err := io.ReadFull(f, s.workingSet) if err != nil { log.Errorf("Reading working set file failed: %v\n", err) @@ -583,394 +275,3 @@ func (s *SnapshotState) fetchWorkingSet() error { log.Debug("Fetched the entire working set") return nil } - -// alignment returns alignment of the block in memory with reference to alignSize. -func alignment(block []byte, alignSize int) int { - return int(uintptr(unsafe.Pointer(&block[0])) & uintptr(alignSize-1)) -} - -// AlignedBlock returns a []byte aligned to the host page size. -func AlignedBlock(blockSize int) []byte { - if blockSize == 0 { - return nil - } - - alignSize := os.Getpagesize() - block := make([]byte, blockSize+alignSize) - - if a := alignment(block, alignSize); a != 0 { - block = block[alignSize-a:] - } - return block[:blockSize] -} - -func (s *SnapshotState) pollUserPageFaults(readyCh chan error) { - logger := log.WithFields(log.Fields{"vmID": s.VMID}) - - var events [2]syscall.EpollEvent - - defer close(s.pollDoneCh) - - if err := s.registerEpoller(); err != nil { - readyCh <- err - return - } - - logger.Debug("Starting polling loop") - - defer func() { _ = syscall.Close(s.epfd) }() - - readyCh <- nil - - for { - select { - case <-s.quitCh: - logger.Debug("Handler received a signal to quit") - return - default: - nevents, err := syscall.EpollWait(s.epfd, events[:], -1) - if err != nil { - if errors.Is(err, syscall.EINTR) { - continue - } - if errors.Is(err, syscall.EBADF) { - logger.Debug("UFFD epoller was closed") - return - } - logger.WithError(err).Error("epoll_wait failed") - return - } - - if nevents < 1 { - continue - } - - select { - case <-s.quitCh: - logger.Debug("Handler received a signal to quit") - return - default: - } - - for i := 0; i < nevents; i++ { - event := events[i] - - fd := int(event.Fd) - if fd == s.wakeFD { - logger.Debug("Handler received wakeup event") - return - } - - stateFd := int(s.userFaultFD.Fd()) - - if fd != stateFd && stateFd != -1 { - logger.WithFields(log.Fields{ - "fd": fd, - "stateFd": stateFd, - }).Error("Received event from unknown fd") - return - } - - goMsg := make([]byte, sizeOfUFFDMsg()) - - nread, err := syscall.Read(fd, goMsg) - if err != nil { - if errors.Is(err, syscall.EINTR) || errors.Is(err, syscall.EAGAIN) { - continue - } - if errors.Is(err, syscall.EBADF) { - logger.Debug("UFFD fd was closed") - return - } - logger.WithError(err).Error("Read uffd_msg failed") - return - } - if nread != len(goMsg) { - logger.WithFields(log.Fields{ - "read": nread, - "want": len(goMsg), - }).Error("Read incomplete uffd_msg") - return - } - - if event := uint8(goMsg[0]); event != uffdPageFault() { - logger.WithField("event", event).Warn("Ignoring unsupported UFFD event") - continue - } - - address := binary.LittleEndian.Uint64(goMsg[16:]) - - if err := s.servePageFault(fd, address); err != nil { - logger.WithError(err).WithField("address", fmt.Sprintf("%#x", address)).Error("Failed to serve page fault") - return - } - } - } - } -} - -func (s *SnapshotState) registerEpoller() error { - logger := log.WithFields(log.Fields{"vmID": s.VMID}) - - var ( - err error - event syscall.EpollEvent - fdInt int - ) - - fdInt = int(s.userFaultFD.Fd()) - - event.Events = syscall.EPOLLIN - event.Fd = int32(fdInt) - - s.epfd, err = syscall.EpollCreate1(0) - if err != nil { - logger.Errorf("Failed to create epoller %v", err) - return err - } - - if err := syscall.EpollCtl( - s.epfd, - syscall.EPOLL_CTL_ADD, - fdInt, - &event, - ); err != nil { - _ = syscall.Close(s.epfd) - logger.Errorf("Failed to subscribe VM %v", err) - return err - } - - s.wakeFD, err = unix.Eventfd(0, unix.EFD_CLOEXEC|unix.EFD_NONBLOCK) - if err != nil { - _ = syscall.Close(s.epfd) - logger.Errorf("Failed to create UFFD wake fd %v", err) - return err - } - - event.Fd = int32(s.wakeFD) - if err := syscall.EpollCtl( - s.epfd, - syscall.EPOLL_CTL_ADD, - s.wakeFD, - &event, - ); err != nil { - _ = unix.Close(s.wakeFD) - _ = syscall.Close(s.epfd) - logger.Errorf("Failed to subscribe UFFD wake fd %v", err) - return err - } - - return nil -} - -func (s *SnapshotState) stopPolling() { - select { - case s.quitCh <- 0: - default: - } - - if s.wakeFD < 0 { - return - } - - var wake [8]byte - binary.LittleEndian.PutUint64(wake[:], 1) - if _, err := unix.Write(s.wakeFD, wake[:]); err != nil && - !errors.Is(err, syscall.EBADF) && - !errors.Is(err, syscall.EAGAIN) { - log.WithError(err).Debug("Failed to wake UFFD poller") - } -} - -func (s *SnapshotState) waitForPoller() { - if s.pollDoneCh != nil { - <-s.pollDoneCh - } -} - -func (s *SnapshotState) closeWakeFD() { - if s.wakeFD >= 0 { - _ = unix.Close(s.wakeFD) - s.wakeFD = -1 - } -} - -func (s *SnapshotState) servePageFault(fd int, address uint64) error { - var ( - tStart time.Time - workingSetInstalled bool - ) - - copyArgs, err := pageFaultCopyArgsForFault(s.guestRegionMappings, address) - if err != nil { - return err - } - - rec := Record{offset: copyArgs.srcOffset} - if s.firstPageFaultOnce != nil { - s.firstPageFaultOnce.Do(func() { - if !s.isRecordReady || s.IsLazyMode { - return - } - - if s.metricsModeOn { - tStart = time.Now() - } - err = s.installWorkingSetPages(fd, copyArgs.dstAddr, copyArgs.copyLen) - if err != nil { - return - } - if s.metricsModeOn { - s.currentMetric.MetricMap[installWSMetric] = metrics.ToUS(time.Since(tStart)) - } - workingSetInstalled = true - }) - if err != nil { - return err - } - } - - if workingSetInstalled && s.trace.containsRecord(rec) { - return nil - } - - src, err := guestMemPointer(s.guestMem, copyArgs.srcOffset, copyArgs.copyLen) - if err != nil { - return err - } - - if !s.isRecordReady { - s.trace.AppendRecord(rec) - } else { - log.Debug("Serving a page that is missing from the working set") - } - - if s.metricsModeOn { - if s.isRecordReady { - if s.IsLazyMode { - if !s.trace.containsRecord(rec) { - s.uniqueNum++ - } - s.replayedNum++ - } else { - s.uniqueNum++ - } - } - tStart = time.Now() - } - - err = installRegionBytes(fd, src, copyArgs.dstAddr, copyArgs.copyMode, copyArgs.copyLen) - - if s.metricsModeOn { - s.currentMetric.MetricMap[serveUniqueMetric] += metrics.ToUS(time.Since(tStart)) - } - - return err -} - -func (s *SnapshotState) installWorkingSetPages(fd int, faultPageAddr, pageSize uint64) error { - if len(s.workingSet) == 0 || len(s.trace.regions) == 0 { - return nil - } - if s.trace.pageSize != 0 { - pageSize = s.trace.pageSize - } - - keys := make([]uint64, 0, len(s.trace.regions)) - for offset := range s.trace.regions { - keys = append(keys, offset) - } - sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) - - var workingSetOffset uint64 - for _, regionOffset := range keys { - regLength := s.trace.regions[regionOffset] - for i := 0; i < regLength; i++ { - pageOffset := regionOffset + uint64(i)*pageSize - copyArgs, err := pageFaultCopyArgsForGuestOffset( - s.guestRegionMappings, - pageOffset, - uint64(C.const_UFFDIO_COPY_MODE_DONTWAKE), - ) - if err != nil { - return err - } - - src, err := guestMemPointer(s.workingSet, workingSetOffset, copyArgs.copyLen) - if err != nil { - return err - } - if err := installRegionBytes(fd, src, copyArgs.dstAddr, copyArgs.copyMode, copyArgs.copyLen); err != nil { - return err - } - workingSetOffset += copyArgs.copyLen - } - } - - return wake(fd, faultPageAddr, pageSize) -} - -func installRegionBytes(fd int, src, dst, mode, length uint64) error { - cUC := C.struct_uffdio_copy{ - mode: C.ulonglong(mode), - copy: 0, - src: C.ulonglong(src), - dst: C.ulonglong(dst), - len: C.ulonglong(length), - } - - err := ioctl(uintptr(fd), int(C.const_UFFDIO_COPY), unsafe.Pointer(&cUC)) - if err != nil { - if errors.Is(err, unix.EEXIST) { - return nil - } - return err - } - - return nil -} - -func guestMemPointer(guestMem []byte, offset, length uint64) (uint64, error) { - if length == 0 { - return 0, errors.New("guest memory copy length must be non-zero") - } - if offset >= uint64(len(guestMem)) || length > uint64(len(guestMem))-offset { - return 0, fmt.Errorf("guest memory copy is outside mapped file: offset=%#x len=%#x size=%#x", offset, length, len(guestMem)) - } - - return uint64(uintptr(unsafe.Pointer(&guestMem[int(offset)]))), nil -} - -func ioctl(fd uintptr, request int, argp unsafe.Pointer) error { - _, _, errno := unix.Syscall( - unix.SYS_IOCTL, - fd, - uintptr(request), - // Note that the conversion from unsafe.Pointer to uintptr _must_ - // occur in the call expression. See the package unsafe documentation - // for more details. - uintptr(argp), - ) - if errno != 0 { - return os.NewSyscallError("ioctl", errno) - } - - return nil -} - -func wake(fd int, startAddress, length uint64) error { - cUR := C.struct_uffdio_range{ - start: C.ulonglong(startAddress), - len: C.ulonglong(length), - } - - return ioctl(uintptr(fd), int(C.const_UFFDIO_WAKE), unsafe.Pointer(&cUR)) -} - -func sizeOfUFFDMsg() int { - return C.sizeof_struct_uffd_msg -} - -func uffdPageFault() uint8 { - return uint8(C.const_UFFD_EVENT_PAGEFAULT) -} diff --git a/memory/manager/uffd.go b/memory/manager/uffd.go new file mode 100644 index 000000000..dd3075425 --- /dev/null +++ b/memory/manager/uffd.go @@ -0,0 +1,741 @@ +// MIT License +// +// Copyright (c) 2020 Dmitrii Ustiugov, Plamen Petrov and EASE lab +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package manager + +/* +#include "user_page_faults.h" +*/ +import "C" + +import ( + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "math" + "net" + "os" + "path/filepath" + "sort" + "syscall" + "time" + "unsafe" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" + + "github.com/vhive-serverless/vhive/metrics" +) + +const ( + uffdSocketPayloadSize = 64 * 1024 + uffdSocketFDLimit = 2 + uffdSocketReadTimeout = time.Second + uffdSocketAcceptTimeout = 30 * time.Second +) + +var ( + errInvalidGuestRegionPageSize = errors.New("guest region page size must be non-zero") + errGuestRegionNotFound = errors.New("fault address is outside guest memory mappings") + errUnexpectedUffdFDCount = errors.New("expected exactly one uffd fd") + errNoGuestRegionMappings = errors.New("no guest region mappings received") + errEmptyUffdSocketPath = errors.New("empty uffd socket path") +) + +// GuestRegionUffdMapping describes Firecracker's UFFD guest memory mapping. +type GuestRegionUffdMapping struct { + BaseHostVirtAddr uint64 `json:"base_host_virt_addr"` + Size uint64 `json:"size"` + Offset uint64 `json:"offset"` + PageSize uint64 `json:"page_size"` +} + +// pageFaultCopyArgs describes one UFFDIO_COPY operation. +type pageFaultCopyArgs struct { + srcOffset uint64 + dstAddr uint64 + copyLen uint64 + copyMode uint64 +} + +// pageAlignFaultAddress rounds a fault address down to its guest page boundary. +func pageAlignFaultAddress(faultAddr uint64, region GuestRegionUffdMapping) (uint64, error) { + if region.PageSize == 0 { + return 0, errInvalidGuestRegionPageSize + } + + return faultAddr - faultAddr%region.PageSize, nil +} + +// guestMemoryOffsetForFaultPage translates a guest fault page to its memory-file offset. +func guestMemoryOffsetForFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) (uint64, error) { + if region.PageSize == 0 { + return 0, errInvalidGuestRegionPageSize + } + if !regionContainsFaultPage(region, faultPageAddr) { + return 0, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultPageAddr) + } + + regionOffset := faultPageAddr - region.BaseHostVirtAddr + if region.Offset > math.MaxUint64-regionOffset { + return 0, fmt.Errorf("guest memory offset overflow for fault address %#x", faultPageAddr) + } + + return region.Offset + regionOffset, nil +} + +// guestAddressForMemoryOffset translates a memory-file offset to a guest address. +func guestAddressForMemoryOffset(region GuestRegionUffdMapping, offset uint64) (uint64, error) { + if region.PageSize == 0 { + return 0, errInvalidGuestRegionPageSize + } + if !regionContainsGuestMemoryOffset(region, offset) { + return 0, fmt.Errorf("%w: %#x", errGuestRegionNotFound, offset) + } + + regionOffset := offset - region.Offset + if region.BaseHostVirtAddr > math.MaxUint64-regionOffset { + return 0, fmt.Errorf("guest address overflow for memory offset %#x", offset) + } + + return region.BaseHostVirtAddr + regionOffset, nil +} + +// pageFaultCopyArgsForFault builds copy arguments for a reported guest page fault. +func pageFaultCopyArgsForFault(regions []GuestRegionUffdMapping, faultAddr uint64) (pageFaultCopyArgs, error) { + for _, region := range regions { + faultPageAddr, err := pageAlignFaultAddress(faultAddr, region) + if err != nil { + return pageFaultCopyArgs{}, err + } + if !regionContainsFaultPage(region, faultPageAddr) { + continue + } + + srcOffset, err := guestMemoryOffsetForFaultPage(region, faultPageAddr) + if err != nil { + return pageFaultCopyArgs{}, err + } + + return pageFaultCopyArgs{ + srcOffset: srcOffset, + dstAddr: faultPageAddr, + copyLen: region.PageSize, + copyMode: 0, + }, nil + } + + return pageFaultCopyArgs{}, fmt.Errorf("%w: %#x", errGuestRegionNotFound, faultAddr) +} + +// pageFaultCopyArgsForGuestOffset builds copy arguments for working-set replay. +func pageFaultCopyArgsForGuestOffset(regions []GuestRegionUffdMapping, offset uint64, mode uint64) (pageFaultCopyArgs, error) { + for _, region := range regions { + if !regionContainsGuestMemoryOffset(region, offset) { + continue + } + dstAddr, err := guestAddressForMemoryOffset(region, offset) + if err != nil { + return pageFaultCopyArgs{}, err + } + + return pageFaultCopyArgs{ + srcOffset: offset, + dstAddr: dstAddr, + copyLen: region.PageSize, + copyMode: mode, + }, nil + } + + return pageFaultCopyArgs{}, fmt.Errorf("%w: %#x", errGuestRegionNotFound, offset) +} + +// regionContainsFaultPage reports whether a guest page belongs to a mapping. +func regionContainsFaultPage(region GuestRegionUffdMapping, faultPageAddr uint64) bool { + if region.Size == 0 || faultPageAddr < region.BaseHostVirtAddr { + return false + } + + return faultPageAddr-region.BaseHostVirtAddr < region.Size +} + +// regionContainsGuestMemoryOffset reports whether a memory-file offset belongs to a mapping. +func regionContainsGuestMemoryOffset(region GuestRegionUffdMapping, offset uint64) bool { + if region.Size == 0 || offset < region.Offset { + return false + } + + return offset-region.Offset < region.Size +} + +// guestMappingPageSize returns the common page size used by all mappings. +func guestMappingPageSize(regions []GuestRegionUffdMapping) (uint64, error) { + var pageSize uint64 + for _, region := range regions { + if region.PageSize == 0 { + return 0, errInvalidGuestRegionPageSize + } + if pageSize == 0 { + pageSize = region.PageSize + continue + } + if pageSize != region.PageSize { + return 0, errors.New("mixed guest region page sizes are not supported for working-set replay") + } + } + if pageSize == 0 { + return 0, errNoGuestRegionMappings + } + return pageSize, nil +} + +// getUFFD receives and stores Firecracker's mappings and userfaultfd. +func (s *SnapshotState) getUFFD(socketReadyCh chan<- struct{}) error { + mappings, userFaultFD, err := receiveUffdMappingsAndFDFromSocket(s.InstanceSockAddr, socketReadyCh) + if err != nil { + log.Error("Failed to receive the uffd and guest memory mappings") + return err + } + + s.guestRegionMappings = mappings + s.userFaultFD = userFaultFD + + return nil +} + +// receiveUffdMappingsAndFDFromSocket accepts one Firecracker UFFD connection. +func receiveUffdMappingsAndFDFromSocket(socketPath string, socketReadyCh chan<- struct{}) ([]GuestRegionUffdMapping, *os.File, error) { + if socketPath == "" { + return nil, nil, errEmptyUffdSocketPath + } + if err := os.MkdirAll(filepath.Dir(socketPath), 0755); err != nil { + return nil, nil, err + } + if err := removeStaleUffdSocket(socketPath); err != nil { + return nil, nil, err + } + + addr := &net.UnixAddr{Name: socketPath, Net: "unix"} + listener, err := net.ListenUnix("unix", addr) + if err != nil { + return nil, nil, err + } + defer func() { _ = listener.Close() }() + defer func() { _ = os.Remove(socketPath) }() + + if err := listener.SetDeadline(time.Now().Add(uffdSocketAcceptTimeout)); err != nil { + return nil, nil, err + } + + notifySocketReady(socketReadyCh) + + conn, err := listener.AcceptUnix() + if err != nil { + return nil, nil, err + } + defer func() { _ = conn.Close() }() + + return receiveUffdMappingsAndFD(conn) +} + +// notifySocketReady signals that Firecracker can connect to the UFFD socket. +func notifySocketReady(socketReadyCh chan<- struct{}) { + if socketReadyCh == nil { + return + } + socketReadyCh <- struct{}{} +} + +// receiveUffdMappingsAndFD reads mappings and exactly one fd from a Unix connection. +func receiveUffdMappingsAndFD(conn *net.UnixConn) ([]GuestRegionUffdMapping, *os.File, error) { + if err := conn.SetReadDeadline(time.Now().Add(uffdSocketReadTimeout)); err != nil { + return nil, nil, err + } + defer func() { _ = conn.SetReadDeadline(time.Time{}) }() + + body := make([]byte, uffdSocketPayloadSize) + oob := make([]byte, unix.CmsgSpace(uffdSocketFDLimit*4)) + + n, oobn, flags, _, err := conn.ReadMsgUnix(body, oob) + if err != nil { + return nil, nil, err + } + if flags&unix.MSG_TRUNC != 0 { + return nil, nil, errors.New("uffd mappings payload was truncated") + } + if flags&unix.MSG_CTRUNC != 0 { + return nil, nil, errors.New("uffd fd control message was truncated") + } + + fds, err := parseUnixRights(oob[:oobn]) + if err != nil { + return nil, nil, err + } + if len(fds) != 1 { + closeFDs(fds) + return nil, nil, fmt.Errorf("%w: got %d", errUnexpectedUffdFDCount, len(fds)) + } + + uffdFile := os.NewFile(uintptr(fds[0]), "userfaultfd") + if uffdFile == nil { + return nil, nil, errors.New("failed to create file for uffd fd") + } + + var mappings []GuestRegionUffdMapping + if err := json.Unmarshal(body[:n], &mappings); err != nil { + _ = uffdFile.Close() + return nil, nil, fmt.Errorf("cannot deserialize memory mappings: %w", err) + } + if len(mappings) == 0 { + _ = uffdFile.Close() + return nil, nil, errNoGuestRegionMappings + } + + return mappings, uffdFile, nil +} + +// parseUnixRights extracts file descriptors from Unix control messages. +func parseUnixRights(oob []byte) ([]int, error) { + scms, err := unix.ParseSocketControlMessage(oob) + if err != nil { + return nil, err + } + + var fds []int + for i := range scms { + rights, err := unix.ParseUnixRights(&scms[i]) + if err != nil { + closeFDs(fds) + return nil, err + } + fds = append(fds, rights...) + } + + return fds, nil +} + +// closeFDs closes file descriptors that cannot be returned to the caller. +func closeFDs(fds []int) { + for _, receivedFD := range fds { + _ = unix.Close(receivedFD) + } +} + +// removeStaleUffdSocket removes an existing socket without deleting other file types. +func removeStaleUffdSocket(socketPath string) error { + info, err := os.Lstat(socketPath) + if err == nil { + if info.Mode()&os.ModeSocket == 0 { + return fmt.Errorf("refusing to remove non-socket uffd path %q", socketPath) + } + return os.Remove(socketPath) + } + if os.IsNotExist(err) { + return nil + } + return err +} + +// pollUserPageFaults reads UFFD events until the handler is stopped or fails. +func (s *SnapshotState) pollUserPageFaults(readyCh chan error) { + logger := log.WithFields(log.Fields{"vmID": s.VMID}) + + var events [2]syscall.EpollEvent + + defer close(s.pollDoneCh) + + if err := s.registerEpoller(); err != nil { + readyCh <- err + return + } + + logger.Debug("Starting polling loop") + + defer func() { _ = syscall.Close(s.epfd) }() + + readyCh <- nil + + for { + select { + case <-s.quitCh: + logger.Debug("Handler received a signal to quit") + return + default: + nevents, err := syscall.EpollWait(s.epfd, events[:], -1) + if err != nil { + if errors.Is(err, syscall.EINTR) { + continue + } + if errors.Is(err, syscall.EBADF) { + logger.Debug("UFFD epoller was closed") + return + } + logger.WithError(err).Error("epoll_wait failed") + return + } + + if nevents < 1 { + continue + } + + select { + case <-s.quitCh: + logger.Debug("Handler received a signal to quit") + return + default: + } + + for i := 0; i < nevents; i++ { + event := events[i] + + fd := int(event.Fd) + if fd == s.wakeFD { + logger.Debug("Handler received wakeup event") + return + } + + stateFd := int(s.userFaultFD.Fd()) + + if fd != stateFd && stateFd != -1 { + logger.WithFields(log.Fields{ + "fd": fd, + "stateFd": stateFd, + }).Error("Received event from unknown fd") + return + } + + goMsg := make([]byte, sizeOfUFFDMsg()) + + nread, err := syscall.Read(fd, goMsg) + if err != nil { + if errors.Is(err, syscall.EINTR) || errors.Is(err, syscall.EAGAIN) { + continue + } + if errors.Is(err, syscall.EBADF) { + logger.Debug("UFFD fd was closed") + return + } + logger.WithError(err).Error("Read uffd_msg failed") + return + } + if nread != len(goMsg) { + logger.WithFields(log.Fields{ + "read": nread, + "want": len(goMsg), + }).Error("Read incomplete uffd_msg") + return + } + + if event := uint8(goMsg[0]); event != uffdPageFault() { + logger.WithField("event", event).Warn("Ignoring unsupported UFFD event") + continue + } + + address := binary.LittleEndian.Uint64(goMsg[16:]) + + if err := s.servePageFault(fd, address); err != nil { + logger.WithError(err).WithField("address", fmt.Sprintf("%#x", address)).Error("Failed to serve page fault") + return + } + } + } + } +} + +// registerEpoller subscribes the UFFD and wake fd to a new epoll instance. +func (s *SnapshotState) registerEpoller() error { + logger := log.WithFields(log.Fields{"vmID": s.VMID}) + + var ( + err error + event syscall.EpollEvent + fdInt int + ) + + fdInt = int(s.userFaultFD.Fd()) + + event.Events = syscall.EPOLLIN + event.Fd = int32(fdInt) + + s.epfd, err = syscall.EpollCreate1(0) + if err != nil { + logger.Errorf("Failed to create epoller %v", err) + return err + } + + if err := syscall.EpollCtl( + s.epfd, + syscall.EPOLL_CTL_ADD, + fdInt, + &event, + ); err != nil { + _ = syscall.Close(s.epfd) + logger.Errorf("Failed to subscribe VM %v", err) + return err + } + + s.wakeFD, err = unix.Eventfd(0, unix.EFD_CLOEXEC|unix.EFD_NONBLOCK) + if err != nil { + _ = syscall.Close(s.epfd) + logger.Errorf("Failed to create UFFD wake fd %v", err) + return err + } + + event.Fd = int32(s.wakeFD) + if err := syscall.EpollCtl( + s.epfd, + syscall.EPOLL_CTL_ADD, + s.wakeFD, + &event, + ); err != nil { + _ = unix.Close(s.wakeFD) + _ = syscall.Close(s.epfd) + logger.Errorf("Failed to subscribe UFFD wake fd %v", err) + return err + } + + return nil +} + +// stopPolling wakes the poller and asks it to exit. +func (s *SnapshotState) stopPolling() { + select { + case s.quitCh <- 0: + default: + } + + if s.wakeFD < 0 { + return + } + + var wake [8]byte + binary.LittleEndian.PutUint64(wake[:], 1) + if _, err := unix.Write(s.wakeFD, wake[:]); err != nil && + !errors.Is(err, syscall.EBADF) && + !errors.Is(err, syscall.EAGAIN) { + log.WithError(err).Debug("Failed to wake UFFD poller") + } +} + +// waitForPoller waits until the UFFD polling goroutine exits. +func (s *SnapshotState) waitForPoller() { + if s.pollDoneCh != nil { + <-s.pollDoneCh + } +} + +// closeWakeFD releases the eventfd used to stop the poller. +func (s *SnapshotState) closeWakeFD() { + if s.wakeFD >= 0 { + _ = unix.Close(s.wakeFD) + s.wakeFD = -1 + } +} + +// servePageFault copies the requested guest page or installs the recorded working set. +func (s *SnapshotState) servePageFault(fd int, address uint64) error { + var ( + tStart time.Time + workingSetInstalled bool + ) + + copyArgs, err := pageFaultCopyArgsForFault(s.guestRegionMappings, address) + if err != nil { + return err + } + + rec := Record{offset: copyArgs.srcOffset} + if s.firstPageFaultOnce != nil { + s.firstPageFaultOnce.Do(func() { + if !s.isRecordReady || s.IsLazyMode { + return + } + + if s.metricsModeOn { + tStart = time.Now() + } + err = s.installWorkingSetPages(fd, copyArgs.dstAddr, copyArgs.copyLen) + if err != nil { + return + } + if s.metricsModeOn { + s.currentMetric.MetricMap[installWSMetric] = metrics.ToUS(time.Since(tStart)) + } + workingSetInstalled = true + }) + if err != nil { + return err + } + } + + if workingSetInstalled && s.trace.containsRecord(rec) { + return nil + } + + src, err := guestMemPointer(s.guestMem, copyArgs.srcOffset, copyArgs.copyLen) + if err != nil { + return err + } + + if !s.isRecordReady { + s.trace.AppendRecord(rec) + } else { + log.Debug("Serving a page that is missing from the working set") + } + + if s.metricsModeOn { + if s.isRecordReady { + if s.IsLazyMode { + if !s.trace.containsRecord(rec) { + s.uniqueNum++ + } + s.replayedNum++ + } else { + s.uniqueNum++ + } + } + tStart = time.Now() + } + + err = installRegionBytes(fd, src, copyArgs.dstAddr, copyArgs.copyMode, copyArgs.copyLen) + + if s.metricsModeOn { + s.currentMetric.MetricMap[serveUniqueMetric] += metrics.ToUS(time.Since(tStart)) + } + + return err +} + +// installWorkingSetPages copies recorded pages before waking the first fault. +func (s *SnapshotState) installWorkingSetPages(fd int, faultPageAddr, pageSize uint64) error { + if len(s.workingSet) == 0 || len(s.trace.regions) == 0 { + return nil + } + if s.trace.pageSize != 0 { + pageSize = s.trace.pageSize + } + + keys := make([]uint64, 0, len(s.trace.regions)) + for offset := range s.trace.regions { + keys = append(keys, offset) + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + + var workingSetOffset uint64 + for _, regionOffset := range keys { + regLength := s.trace.regions[regionOffset] + for i := 0; i < regLength; i++ { + pageOffset := regionOffset + uint64(i)*pageSize + copyArgs, err := pageFaultCopyArgsForGuestOffset( + s.guestRegionMappings, + pageOffset, + uint64(C.const_UFFDIO_COPY_MODE_DONTWAKE), + ) + if err != nil { + return err + } + + src, err := guestMemPointer(s.workingSet, workingSetOffset, copyArgs.copyLen) + if err != nil { + return err + } + if err := installRegionBytes(fd, src, copyArgs.dstAddr, copyArgs.copyMode, copyArgs.copyLen); err != nil { + return err + } + workingSetOffset += copyArgs.copyLen + } + } + + return wake(fd, faultPageAddr, pageSize) +} + +// installRegionBytes resolves missing pages with UFFDIO_COPY. +func installRegionBytes(fd int, src, dst, mode, length uint64) error { + cUC := C.struct_uffdio_copy{ + mode: C.ulonglong(mode), + copy: 0, + src: C.ulonglong(src), + dst: C.ulonglong(dst), + len: C.ulonglong(length), + } + + err := ioctl(uintptr(fd), int(C.const_UFFDIO_COPY), unsafe.Pointer(&cUC)) + if err != nil { + if errors.Is(err, unix.EEXIST) { + return nil + } + return err + } + + return nil +} + +// guestMemPointer returns a checked pointer into a mapped memory buffer. +func guestMemPointer(guestMem []byte, offset, length uint64) (uint64, error) { + if length == 0 { + return 0, errors.New("guest memory copy length must be non-zero") + } + if offset >= uint64(len(guestMem)) || length > uint64(len(guestMem))-offset { + return 0, fmt.Errorf("guest memory copy is outside mapped file: offset=%#x len=%#x size=%#x", offset, length, len(guestMem)) + } + + return uint64(uintptr(unsafe.Pointer(&guestMem[int(offset)]))), nil +} + +// ioctl invokes an ioctl and converts errno to a Go error. +func ioctl(fd uintptr, request int, argp unsafe.Pointer) error { + _, _, errno := unix.Syscall( + unix.SYS_IOCTL, + fd, + uintptr(request), + // Note that the conversion from unsafe.Pointer to uintptr _must_ + // occur in the call expression. See the package unsafe documentation + // for more details. + uintptr(argp), + ) + if errno != 0 { + return os.NewSyscallError("ioctl", errno) + } + + return nil +} + +// wake resumes faults waiting on a previously copied range. +func wake(fd int, startAddress, length uint64) error { + cUR := C.struct_uffdio_range{ + start: C.ulonglong(startAddress), + len: C.ulonglong(length), + } + + return ioctl(uintptr(fd), int(C.const_UFFDIO_WAKE), unsafe.Pointer(&cUR)) +} + +// sizeOfUFFDMsg returns the platform uffd_msg size. +func sizeOfUFFDMsg() int { + return C.sizeof_struct_uffd_msg +} + +// uffdPageFault returns the platform page-fault event identifier. +func uffdPageFault() uint8 { + return uint8(C.const_UFFD_EVENT_PAGEFAULT) +}