fix: metadata not reachable on no ip bridge

pkg/agent/server/hostlocal.go

@@ -38,21 +38,14 @@ func NewHostLocal(watcher *serversWatcher) *HostLocal {
 func (hl *HostLocal) updateFlows(ctx context.Context) {
 	for _, hcn := range hl.watcher.hostConfig.HostNetworkConfigs() {
 		log.Debugf("update flows for %s", hcn.Bridge)
-		mac, err := hcn.MAC()
+		_, err := hcn.MAC()
 		if err != nil {
 			log.Errorf("get ip/mac for %s failed: %s", hcn.Bridge, err)
 			continue
 		}
 		hostLocal := &utils.HostLocal{
-			HostConfig: hl.watcher.hostConfig,
-			Bridge:     hcn.Bridge,
-			Ifname:     hcn.Ifname,
-			IP:         hcn.IP,
-			IP6:        hcn.IP6,
-			IP6Local:   hcn.IP6Local,
-			MAC:        mac,
-
-			HostLocalNets: hcn.HostLocalNets,
+			HostConfig:        hl.watcher.hostConfig,
+			HostConfigNetwork: hcn,
 		}
 		hostLocal = utils.FetchHostLocal(hostLocal, hl.watcher)
 
@@ -66,7 +59,7 @@ func (hl *HostLocal) updateFlows(ctx context.Context) {
 			flowman.updateFlows(ctx, hostLocal.Who(), flows[hcn.Bridge])
 		}
 
-		if hostLocal.IP != nil {
+		if hostLocal.IP4() != nil {
 			err = hostLocal.EnsureFakeLocalMetadataRoute()
 			if err != nil {
 				log.Errorf("EnsureFakeLocalMetadataRoute %s hostlocal flows failed: %s", hcn.Bridge, err)

pkg/agent/server/service.go

@@ -58,9 +58,17 @@ func StartService() {
 	)
 	ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "sdnagent")
 	if hc, err = utils.NewHostConfig(); err != nil {
-		log.Errorln(errors.Wrap(err, "host config"))
-	} else if err = hc.Auth(ctx); err != nil {
-		log.Errorln(errors.Wrap(err, "keystone auth"))
+		log.Fatalln(errors.Wrap(err, "host config"))
+	} else {
+		err := hc.WaitMacReady()
+		if err != nil {
+			log.Fatalln(errors.Wrap(err, "wait addr ready"))
+		}
+		log.Infof("host config all networks mac ready")
+	}
+
+	if err = hc.Auth(ctx); err != nil {
+		log.Fatalln(errors.Wrap(err, "keystone auth"))
 	}
 
 	{

pkg/agent/server/watch.go

@@ -462,9 +462,9 @@ func (w *serversWatcher) watchEvent(ev *fsnotify.Event) (wev *watchEvent) {
 	return nil
 }
 
-func (w *serversWatcher) GetHostLocalByIp(ip string) *utils.HostLocal {
+func (w *serversWatcher) GetHostLocalByIp4(ip string) *utils.HostLocal {
 	for _, hl := range w.hostLocal.bridgeMap {
-		if hl.IP.String() == ip {
+		if hl.IP4() != nil && hl.IP4().String() == ip {
 			return hl
 		}
 	}

pkg/agent/utils/cls_md.go

@@ -29,7 +29,7 @@ import (
 )
 
 func (h *HostLocal) fakeMdSrcIpMac(port int) (string, string, string) {
-	return fakeMdSrcIpMac(h.IP.String(), h.MAC.String(), port)
+	return fakeMdSrcIpMac(h.IP4().String(), h.mac.String(), port)
 }
 
 func fakeMdSrcIpMac(ip, mac string, port int) (string, string, string) {
@@ -52,7 +52,7 @@ func fakeMdSrcIpMac(ip, mac string, port int) (string, string, string) {
 }
 
 func (h *HostLocal) fakeMdSrcIp6Mac(port int, metaSrvIp6 string) (string, string, string) {
-	return fakeMdSrcIp6Mac(h.IP6Local.String(), h.MAC.String(), port, metaSrvIp6)
+	return fakeMdSrcIp6Mac(h.IP6Local.String(), h.mac.String(), port, metaSrvIp6)
 }
 
 func mergeByte(a, b byte) uint16 {
@@ -81,7 +81,7 @@ func fakeMdSrcIp6Mac(ip6, mac string, port int, metaSrvIp6 string) (string, stri
 }
 
 func (h *HostLocal) StartMetadataServer(watcher IServerWatcher) {
-	var addr string
+	addr := "" // listen for both ipv4 and ipv6
 	sport := h.HostConfig.MetadataPort() + 1
 	for port := sport; port < sport+64; port++ {
 		if !netutils2.IsTcpPortUsed(addr, port) {
@@ -100,12 +100,11 @@ func (h *HostLocal) StartMetadataServer(watcher IServerWatcher) {
 			watcher:   watcher,
 		},
 	}
-	dbAccess := false
 	h.metadataApp = app.InitApp(&common_options.BaseOptions{
 		ApplicationID:          "metadata-server-class-network",
 		RequestWorkerCount:     4,
 		RequestWorkerQueueSize: 128,
-	}, dbAccess)
+	}, false)
 	log.Infof("Start metadata server at %s on port %s:%d", h.Bridge, addr, h.metadataPort)
 	metadata.Start(h.metadataApp, svc)
 }

pkg/agent/utils/flowsource.go

@@ -141,11 +141,11 @@ func (h *HostLocal) FlowsMap() (map[string][]*ovs.Flow, error) {
 	}
 	m := map[string]interface{}{
 		"MetadataPort": h.metadataPort,
-		"MAC":          h.MAC,
+		"MAC":          h.mac.String(),
 		"PortNoPhy":    portNoPhy,
 	}
-	if h.IP != nil {
-		m["IP"] = h.IP
+	if h.IP4() != nil {
+		m["IP"] = h.IP4()
 	}
 	if h.IP6 != nil {
 		m["IP6"] = h.IP6
@@ -166,20 +166,21 @@ func (h *HostLocal) FlowsMap() (map[string][]*ovs.Flow, error) {
 			F(0, 26900, T("in_port={{.PortNoPhy}},dl_dst={{.MAC}}"), "normal"),
 		)
 	}
-	if h.IP != nil {
+	if h.IP != nil || h.IPLocal != nil {
 		flows = append(flows,
 			// drop arp request from outside to metadata IPv4 address
 			F(0, 29312, T("in_port={{.PortNoPhy}},arp,arp_op=1,arp_tpa=169.254.169.254"), "drop"),
 			// arp response to metadata IPv4 address for guest
-			F(0, 29311, "arp,arp_op=1,arp_tpa=169.254.169.254", FakeArpRespActions(h.MAC.String())),
+			F(0, 29311, "arp,arp_op=1,arp_tpa=169.254.169.254", FakeArpRespActions(h.mac.String())),
 		)
 		// direct all ipv4 metadata response to table 12
 		prefix, _, mac := h.fakeMdSrcIpMac(0)
+		flowMatch := fmt.Sprintf("in_port=LOCAL,tcp,dl_dst=%s,nw_dst=%s,tp_src=%d", mac, prefix, h.metadataPort)
 		flows = append(flows,
-			F(0, 29310, fmt.Sprintf("in_port=LOCAL,tcp,dl_dst=%s,nw_dst=%s,tp_src=%d", mac, prefix, h.metadataPort), "resubmit(,12)"),
+			F(0, 29310, flowMatch, "resubmit(,12)"),
 		)
 	}
-	if h.IP6 != nil {
+	if h.IP6 != nil || h.IP6Local != nil {
 		// drop nbp solicitation from outside to IPv6 metadata  address
 		for i := range h.HostConfig.MetadataServerIp6s {
 			metaSrvIp6 := h.HostConfig.MetadataServerIp6s[i]
@@ -540,6 +541,8 @@ func (g *Guest) FlowsMapForNic(nic *GuestNIC) (map[string][]*ovs.Flow, error) {
 		}
 		if hcn.IP != nil {
 			m["IPPhy"] = hcn.IP.String()
+		} else if hcn.IPLocal != nil {
+			m["IPPhy"] = hcn.IPLocal.String()
 		}
 		if hcn.IP6Local != nil {
 			// ipv6 use link local address to servce metadata service

pkg/agent/utils/hostconfig.go

@@ -41,12 +41,18 @@ type HostConfigNetwork struct {
 	IP     net.IP
 	mac    net.HardwareAddr
 
+	IPLocal net.IP
+
 	IP6      net.IP
 	IP6Local net.IP
 
 	HostLocalNets []apis.NetworkDetails
 }
 
+func (hcn *HostConfigNetwork) String() string {
+	return fmt.Sprintf("HostConfigNetwork: Bridge=%s, Ifname=%s, IP=%s, IPLocal=%s, IP6=%s, IP6Local=%s", hcn.Bridge, hcn.Ifname, hcn.IP, hcn.IPLocal, hcn.IP6, hcn.IP6Local)
+}
+
 func (hcn *HostConfigNetwork) IpAddr() string {
 	if hcn.IP != nil {
 		return hcn.IP.String()
@@ -103,7 +109,7 @@ func NewHostConfigNetwork(network string) (*HostConfigNetwork, error) {
 }
 
 func (hcn *HostConfigNetwork) MAC() (net.HardwareAddr, error) {
-	if hcn.mac == nil {
+	if (hcn.IP == nil && hcn.IPLocal == nil) || hcn.mac == nil {
 		netif := netutils2.NewNetInterfaceWithExpectIp(hcn.Bridge, hcn.IpAddr(), hcn.Ip6Addr(), hcn.HostLocalGatewayIps())
 		if !netif.Exist() {
 			return nil, errors.Wrapf(errors.ErrNotFound, "net interface %s not found", hcn.Bridge)
@@ -118,7 +124,12 @@ func (hcn *HostConfigNetwork) MAC() (net.HardwareAddr, error) {
 		if len(netif.Addr) > 0 {
 			hcn.IP = net.ParseIP(netif.Addr)
 		}
+		if len(netif.Addr4LinkLocal) > 0 {
+			hcn.IPLocal = net.ParseIP(netif.Addr4LinkLocal)
+		}
 		hcn.mac = netif.GetHardwareAddr()
+
+		log.Infoln(hcn.String())
 	}
 	if hcn.mac != nil {
 		return hcn.mac, nil
@@ -214,6 +225,34 @@ func NewHostConfig() (*HostConfig, error) {
 	return hc, nil
 }
 
+func (hc *HostConfig) WaitMacReady() error {
+	ready := false
+	const TIMEOUT = 5 * 60 * time.Second // 5 minutes
+	startTime := time.Now()
+	for !ready && time.Since(startTime) < TIMEOUT {
+		err := hc.checkMacReady()
+		if err != nil {
+			time.Sleep(1 * time.Second)
+		} else {
+			ready = true
+		}
+	}
+	if !ready {
+		return errors.Wrapf(errors.ErrTimeout, "wait mac ready timeout")
+	}
+	return nil
+}
+
+func (hc *HostConfig) checkMacReady() error {
+	for _, network := range hc.networks {
+		_, err := network.MAC()
+		if err != nil {
+			return errors.Wrapf(err, "wait addr ready for network %s", network.Bridge)
+		}
+	}
+	return nil
+}
+
 func (hc *HostConfig) GetOverlayMTU() int {
 	mtu := hc.OvnUnderlayMtu
 	if mtu < 576 {

pkg/agent/utils/hostlocal.go

@@ -15,30 +15,40 @@
 package utils
 
 import (
+	"fmt"
 	"net"
 	"sync"
 
-	"yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/appsrv"
 )
 
 var hostLocalMap *sync.Map
 
 type HostLocal struct {
 	HostConfig *HostConfig
-	Bridge     string
-	Ifname     string
-	IP         net.IP
-	IP6        net.IP
-	IP6Local   net.IP
-	MAC        net.HardwareAddr
 
-	HostLocalNets []compute.NetworkDetails
+	*HostConfigNetwork
 
 	metadataPort int
 	metadataApp  *appsrv.Application
 }
 
+func (h *HostLocal) String() string {
+	prefix, mdIp, mdMac := h.fakeMdSrcIpMac(0)
+	prefix6, mdIp6, mdMac6 := h.fakeMdSrcIp6Mac(0, "")
+	return fmt.Sprintf("HostLocal: Bridge=%s, ifname=%s, IP=%s, IPLocal=%s, IP6=%s, IP6Local=%s, MAC=%s, fakeMdPrefix=%s fakeMdIp=%s fakeMdPrefix6=%s fakeMdSrcIpMac=%s fakeMdIp6=%s fakeMdSrcIp6Mac=%s", h.Bridge, h.Ifname, h.IP, h.IPLocal, h.IP6, h.IP6Local, h.mac, prefix, mdIp, mdMac, prefix6, mdIp6, mdMac6)
+}
+
+func (h *HostLocal) IP4() net.IP {
+	if h.IP != nil {
+		return h.IP
+	}
+	if h.IPLocal != nil {
+		return h.IPLocal
+	}
+	return nil
+}
+
 func init() {
 	hostLocalMap = &sync.Map{}
 }