From c879eaf5f3c49f89ea5308b1ab8bb306717785b6 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 26 Jun 2026 11:34:52 +0800 Subject: [PATCH] fix: metadata not reachable on no ip bridge --- pkg/agent/server/hostlocal.go | 15 ++++--------- pkg/agent/server/service.go | 14 +++++++++--- pkg/agent/server/watch.go | 4 ++-- pkg/agent/utils/cls_md.go | 9 ++++---- pkg/agent/utils/flowsource.go | 17 +++++++++------ pkg/agent/utils/hostconfig.go | 41 ++++++++++++++++++++++++++++++++++- pkg/agent/utils/hostlocal.go | 26 +++++++++++++++------- 7 files changed, 89 insertions(+), 37 deletions(-) diff --git a/pkg/agent/server/hostlocal.go b/pkg/agent/server/hostlocal.go index e7641d79a..0cc9c9a28 100644 --- a/pkg/agent/server/hostlocal.go +++ b/pkg/agent/server/hostlocal.go @@ -38,21 +38,14 @@ func NewHostLocal(watcher *serversWatcher) *HostLocal { func (hl *HostLocal) updateFlows(ctx context.Context) { for _, hcn := range hl.watcher.hostConfig.HostNetworkConfigs() { log.Debugf("update flows for %s", hcn.Bridge) - mac, err := hcn.MAC() + _, err := hcn.MAC() if err != nil { log.Errorf("get ip/mac for %s failed: %s", hcn.Bridge, err) continue } hostLocal := &utils.HostLocal{ - HostConfig: hl.watcher.hostConfig, - Bridge: hcn.Bridge, - Ifname: hcn.Ifname, - IP: hcn.IP, - IP6: hcn.IP6, - IP6Local: hcn.IP6Local, - MAC: mac, - - HostLocalNets: hcn.HostLocalNets, + HostConfig: hl.watcher.hostConfig, + HostConfigNetwork: hcn, } hostLocal = utils.FetchHostLocal(hostLocal, hl.watcher) @@ -66,7 +59,7 @@ func (hl *HostLocal) updateFlows(ctx context.Context) { flowman.updateFlows(ctx, hostLocal.Who(), flows[hcn.Bridge]) } - if hostLocal.IP != nil { + if hostLocal.IP4() != nil { err = hostLocal.EnsureFakeLocalMetadataRoute() if err != nil { log.Errorf("EnsureFakeLocalMetadataRoute %s hostlocal flows failed: %s", hcn.Bridge, err) diff --git a/pkg/agent/server/service.go b/pkg/agent/server/service.go index b01a21fa2..38df5777e 100644 --- a/pkg/agent/server/service.go +++ b/pkg/agent/server/service.go @@ -58,9 +58,17 @@ func StartService() { ) ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "sdnagent") if hc, err = utils.NewHostConfig(); err != nil { - log.Errorln(errors.Wrap(err, "host config")) - } else if err = hc.Auth(ctx); err != nil { - log.Errorln(errors.Wrap(err, "keystone auth")) + log.Fatalln(errors.Wrap(err, "host config")) + } else { + err := hc.WaitMacReady() + if err != nil { + log.Fatalln(errors.Wrap(err, "wait addr ready")) + } + log.Infof("host config all networks mac ready") + } + + if err = hc.Auth(ctx); err != nil { + log.Fatalln(errors.Wrap(err, "keystone auth")) } { diff --git a/pkg/agent/server/watch.go b/pkg/agent/server/watch.go index 17f555fb9..a446ae725 100644 --- a/pkg/agent/server/watch.go +++ b/pkg/agent/server/watch.go @@ -462,9 +462,9 @@ func (w *serversWatcher) watchEvent(ev *fsnotify.Event) (wev *watchEvent) { return nil } -func (w *serversWatcher) GetHostLocalByIp(ip string) *utils.HostLocal { +func (w *serversWatcher) GetHostLocalByIp4(ip string) *utils.HostLocal { for _, hl := range w.hostLocal.bridgeMap { - if hl.IP.String() == ip { + if hl.IP4() != nil && hl.IP4().String() == ip { return hl } } diff --git a/pkg/agent/utils/cls_md.go b/pkg/agent/utils/cls_md.go index ab06b3d92..10368e904 100644 --- a/pkg/agent/utils/cls_md.go +++ b/pkg/agent/utils/cls_md.go @@ -29,7 +29,7 @@ import ( ) func (h *HostLocal) fakeMdSrcIpMac(port int) (string, string, string) { - return fakeMdSrcIpMac(h.IP.String(), h.MAC.String(), port) + return fakeMdSrcIpMac(h.IP4().String(), h.mac.String(), port) } func fakeMdSrcIpMac(ip, mac string, port int) (string, string, string) { @@ -52,7 +52,7 @@ func fakeMdSrcIpMac(ip, mac string, port int) (string, string, string) { } func (h *HostLocal) fakeMdSrcIp6Mac(port int, metaSrvIp6 string) (string, string, string) { - return fakeMdSrcIp6Mac(h.IP6Local.String(), h.MAC.String(), port, metaSrvIp6) + return fakeMdSrcIp6Mac(h.IP6Local.String(), h.mac.String(), port, metaSrvIp6) } func mergeByte(a, b byte) uint16 { @@ -81,7 +81,7 @@ func fakeMdSrcIp6Mac(ip6, mac string, port int, metaSrvIp6 string) (string, stri } func (h *HostLocal) StartMetadataServer(watcher IServerWatcher) { - var addr string + addr := "" // listen for both ipv4 and ipv6 sport := h.HostConfig.MetadataPort() + 1 for port := sport; port < sport+64; port++ { if !netutils2.IsTcpPortUsed(addr, port) { @@ -100,12 +100,11 @@ func (h *HostLocal) StartMetadataServer(watcher IServerWatcher) { watcher: watcher, }, } - dbAccess := false h.metadataApp = app.InitApp(&common_options.BaseOptions{ ApplicationID: "metadata-server-class-network", RequestWorkerCount: 4, RequestWorkerQueueSize: 128, - }, dbAccess) + }, false) log.Infof("Start metadata server at %s on port %s:%d", h.Bridge, addr, h.metadataPort) metadata.Start(h.metadataApp, svc) } diff --git a/pkg/agent/utils/flowsource.go b/pkg/agent/utils/flowsource.go index fcbe5dd97..60328b1c3 100644 --- a/pkg/agent/utils/flowsource.go +++ b/pkg/agent/utils/flowsource.go @@ -141,11 +141,11 @@ func (h *HostLocal) FlowsMap() (map[string][]*ovs.Flow, error) { } m := map[string]interface{}{ "MetadataPort": h.metadataPort, - "MAC": h.MAC, + "MAC": h.mac.String(), "PortNoPhy": portNoPhy, } - if h.IP != nil { - m["IP"] = h.IP + if h.IP4() != nil { + m["IP"] = h.IP4() } if h.IP6 != nil { m["IP6"] = h.IP6 @@ -166,20 +166,21 @@ func (h *HostLocal) FlowsMap() (map[string][]*ovs.Flow, error) { F(0, 26900, T("in_port={{.PortNoPhy}},dl_dst={{.MAC}}"), "normal"), ) } - if h.IP != nil { + if h.IP != nil || h.IPLocal != nil { flows = append(flows, // drop arp request from outside to metadata IPv4 address F(0, 29312, T("in_port={{.PortNoPhy}},arp,arp_op=1,arp_tpa=169.254.169.254"), "drop"), // arp response to metadata IPv4 address for guest - F(0, 29311, "arp,arp_op=1,arp_tpa=169.254.169.254", FakeArpRespActions(h.MAC.String())), + F(0, 29311, "arp,arp_op=1,arp_tpa=169.254.169.254", FakeArpRespActions(h.mac.String())), ) // direct all ipv4 metadata response to table 12 prefix, _, mac := h.fakeMdSrcIpMac(0) + flowMatch := fmt.Sprintf("in_port=LOCAL,tcp,dl_dst=%s,nw_dst=%s,tp_src=%d", mac, prefix, h.metadataPort) flows = append(flows, - F(0, 29310, fmt.Sprintf("in_port=LOCAL,tcp,dl_dst=%s,nw_dst=%s,tp_src=%d", mac, prefix, h.metadataPort), "resubmit(,12)"), + F(0, 29310, flowMatch, "resubmit(,12)"), ) } - if h.IP6 != nil { + if h.IP6 != nil || h.IP6Local != nil { // drop nbp solicitation from outside to IPv6 metadata address for i := range h.HostConfig.MetadataServerIp6s { metaSrvIp6 := h.HostConfig.MetadataServerIp6s[i] @@ -540,6 +541,8 @@ func (g *Guest) FlowsMapForNic(nic *GuestNIC) (map[string][]*ovs.Flow, error) { } if hcn.IP != nil { m["IPPhy"] = hcn.IP.String() + } else if hcn.IPLocal != nil { + m["IPPhy"] = hcn.IPLocal.String() } if hcn.IP6Local != nil { // ipv6 use link local address to servce metadata service diff --git a/pkg/agent/utils/hostconfig.go b/pkg/agent/utils/hostconfig.go index 74f9dd5b1..007b7649b 100644 --- a/pkg/agent/utils/hostconfig.go +++ b/pkg/agent/utils/hostconfig.go @@ -41,12 +41,18 @@ type HostConfigNetwork struct { IP net.IP mac net.HardwareAddr + IPLocal net.IP + IP6 net.IP IP6Local net.IP HostLocalNets []apis.NetworkDetails } +func (hcn *HostConfigNetwork) String() string { + return fmt.Sprintf("HostConfigNetwork: Bridge=%s, Ifname=%s, IP=%s, IPLocal=%s, IP6=%s, IP6Local=%s", hcn.Bridge, hcn.Ifname, hcn.IP, hcn.IPLocal, hcn.IP6, hcn.IP6Local) +} + func (hcn *HostConfigNetwork) IpAddr() string { if hcn.IP != nil { return hcn.IP.String() @@ -103,7 +109,7 @@ func NewHostConfigNetwork(network string) (*HostConfigNetwork, error) { } func (hcn *HostConfigNetwork) MAC() (net.HardwareAddr, error) { - if hcn.mac == nil { + if (hcn.IP == nil && hcn.IPLocal == nil) || hcn.mac == nil { netif := netutils2.NewNetInterfaceWithExpectIp(hcn.Bridge, hcn.IpAddr(), hcn.Ip6Addr(), hcn.HostLocalGatewayIps()) if !netif.Exist() { return nil, errors.Wrapf(errors.ErrNotFound, "net interface %s not found", hcn.Bridge) @@ -118,7 +124,12 @@ func (hcn *HostConfigNetwork) MAC() (net.HardwareAddr, error) { if len(netif.Addr) > 0 { hcn.IP = net.ParseIP(netif.Addr) } + if len(netif.Addr4LinkLocal) > 0 { + hcn.IPLocal = net.ParseIP(netif.Addr4LinkLocal) + } hcn.mac = netif.GetHardwareAddr() + + log.Infoln(hcn.String()) } if hcn.mac != nil { return hcn.mac, nil @@ -214,6 +225,34 @@ func NewHostConfig() (*HostConfig, error) { return hc, nil } +func (hc *HostConfig) WaitMacReady() error { + ready := false + const TIMEOUT = 5 * 60 * time.Second // 5 minutes + startTime := time.Now() + for !ready && time.Since(startTime) < TIMEOUT { + err := hc.checkMacReady() + if err != nil { + time.Sleep(1 * time.Second) + } else { + ready = true + } + } + if !ready { + return errors.Wrapf(errors.ErrTimeout, "wait mac ready timeout") + } + return nil +} + +func (hc *HostConfig) checkMacReady() error { + for _, network := range hc.networks { + _, err := network.MAC() + if err != nil { + return errors.Wrapf(err, "wait addr ready for network %s", network.Bridge) + } + } + return nil +} + func (hc *HostConfig) GetOverlayMTU() int { mtu := hc.OvnUnderlayMtu if mtu < 576 { diff --git a/pkg/agent/utils/hostlocal.go b/pkg/agent/utils/hostlocal.go index ad16e3c87..2ac698336 100644 --- a/pkg/agent/utils/hostlocal.go +++ b/pkg/agent/utils/hostlocal.go @@ -15,10 +15,10 @@ package utils import ( + "fmt" "net" "sync" - "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/appsrv" ) @@ -26,19 +26,29 @@ var hostLocalMap *sync.Map type HostLocal struct { HostConfig *HostConfig - Bridge string - Ifname string - IP net.IP - IP6 net.IP - IP6Local net.IP - MAC net.HardwareAddr - HostLocalNets []compute.NetworkDetails + *HostConfigNetwork metadataPort int metadataApp *appsrv.Application } +func (h *HostLocal) String() string { + prefix, mdIp, mdMac := h.fakeMdSrcIpMac(0) + prefix6, mdIp6, mdMac6 := h.fakeMdSrcIp6Mac(0, "") + return fmt.Sprintf("HostLocal: Bridge=%s, ifname=%s, IP=%s, IPLocal=%s, IP6=%s, IP6Local=%s, MAC=%s, fakeMdPrefix=%s fakeMdIp=%s fakeMdPrefix6=%s fakeMdSrcIpMac=%s fakeMdIp6=%s fakeMdSrcIp6Mac=%s", h.Bridge, h.Ifname, h.IP, h.IPLocal, h.IP6, h.IP6Local, h.mac, prefix, mdIp, mdMac, prefix6, mdIp6, mdMac6) +} + +func (h *HostLocal) IP4() net.IP { + if h.IP != nil { + return h.IP + } + if h.IPLocal != nil { + return h.IPLocal + } + return nil +} + func init() { hostLocalMap = &sync.Map{} }