TLS prototype

cmd/neofs-node/config.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -33,6 +35,7 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/morph/event"
 	"github.com/nspcc-dev/neofs-node/pkg/network"
 	"github.com/nspcc-dev/neofs-node/pkg/network/cache"
+	"github.com/nspcc-dev/neofs-node/pkg/network/peerauth"
 	"github.com/nspcc-dev/neofs-node/pkg/services/control"
 	controlSvc "github.com/nspcc-dev/neofs-node/pkg/services/control/server"
 	"github.com/nspcc-dev/neofs-node/pkg/services/meta"
@@ -160,6 +163,10 @@ type shared struct {
 	putClientCache *cache.Clients
 	localAddr      network.AddressGroup
 
+	// tlsCert is the node's self-signed certificate built from its identity
+	// private key for inter-node mTLS handshakes.
+	tlsCert tls.Certificate
+
 	ownerIDFromKey user.ID // user ID calculated from key
 
 	// current network map
@@ -414,13 +421,19 @@ func initCfg(appCfg *config.Config) *cfg {
 	fatalOnErr(err)
 
 	basicSharedConfig := initBasics(c, key, persistate)
+
+	tlsCert, err := peerauth.GenerateSelfSignedCert(&key.PrivateKey)
+	fatalOnErr(err)
+	c.log.Info("mTLS: generated self-signed certificate from node identity key",
+		zap.String("pubkey", hex.EncodeToString(key.PublicKey().Bytes())))
+
 	streamTimeout := appCfg.APIClient.StreamTimeout
 	minConnTimeout := appCfg.APIClient.MinConnectionTime
 	pingInterval := appCfg.APIClient.PingInterval
 	pingTimeout := appCfg.APIClient.PingTimeout
 	newClientCache := func(scope string) *cache.Clients {
 		return cache.NewClients(c.log.With(zap.String("scope", scope)), &buffers, streamTimeout,
-			minConnTimeout, pingInterval, pingTimeout, neofsecdsa.Signer(key.PrivateKey))
+			minConnTimeout, pingInterval, pingTimeout, neofsecdsa.Signer(key.PrivateKey), tlsCert)
 	}
 	c.shared = shared{
 		basics:            basicSharedConfig,
@@ -430,6 +443,7 @@ func initCfg(appCfg *config.Config) *cfg {
 		putClientCache:    newClientCache("put"),
 		persistate:        persistate,
 		privateTokenStore: persistate,
+		tlsCert:           tlsCert,
 	}
 	c.cfgBalance = cfgBalance{
 		parsers:     make(map[event.Type]event.NotificationParser),

cmd/neofs-node/grpc.go

@@ -19,7 +19,6 @@ import (
 	"go.uber.org/zap"
 	"golang.org/x/net/netutil"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/resolver"
 )
@@ -185,30 +184,17 @@ func buildSingleGRPCServer(c *cfg, sc grpcconfig.GRPC, maxRecvMsgSizeOpt grpc.Se
 		serverOpts = append(serverOpts, maxRecvMsgSizeOpt)
 	}
 
-	tlsCfg := sc.TLS
-
-	if tlsCfg.Key != "" {
-		certFile, keyFile := tlsCfg.Certificate, tlsCfg.Key
-
-		if _, err := tls.LoadX509KeyPair(certFile, keyFile); err != nil {
+	if sc.TLS.Key != "" {
+		if _, err := tls.LoadX509KeyPair(sc.TLS.Certificate, sc.TLS.Key); err != nil {
 			c.log.Error("could not read certificate from file", zap.Error(err))
 			return nil, nil, err
 		}
 
-		// read certificate from disk on each handshake to pick up renewals automatically.
-		creds := credentials.NewTLS(&tls.Config{
-			GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) {
-				cert, err := tls.LoadX509KeyPair(certFile, keyFile)
-				if err != nil {
-					return nil, fmt.Errorf("reload TLS certificate: %w", err)
-				}
-				return &tls.Config{
-					Certificates: []tls.Certificate{cert},
-				}, nil
-			},
-		})
-
-		serverOpts = append(serverOpts, grpc.Creds(creds))
+		serverOpts = append(serverOpts, grpc.Creds(newNodeAuthCreds(c, sc.TLS.Certificate, sc.TLS.Key)))
+		c.log.Info("gRPC endpoint serves clients (server-side TLS) and inter-node mTLS", zap.String("endpoint", sc.Endpoint))
+	} else {
+		serverOpts = append(serverOpts, grpc.Creds(newNodeAuthCreds(c, "", "")))
+		c.log.Info("gRPC endpoint serves clients (plain) and inter-node mTLS", zap.String("endpoint", sc.Endpoint))
 	}
 
 	lis, err := net.Listen("tcp", sc.Endpoint)

cmd/neofs-node/mtls.go

@@ -0,0 +1,159 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+
+	"github.com/nspcc-dev/neofs-node/pkg/network/peerauth"
+	"github.com/nspcc-dev/neofs-sdk-go/netmap"
+	"go.uber.org/zap"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// tlsRecordTypeHandshake is the first byte (ContentType) of a TLS handshake
+// record.
+const tlsRecordTypeHandshake = 0x16
+
+// nodeAuthCreds are gRPC server transport credentials that serve regular clients
+// and inter-node mTLS connections on the same port.
+type nodeAuthCreds struct {
+	tls   credentials.TransportCredentials
+	plain credentials.TransportCredentials
+}
+
+// newNodeAuthCreds builds the shared-port credentials. adminCertFile and
+// adminKeyFile are the optional server-side TLS certificate served to regular
+// clients; when empty, only plain clients and inter-node mTLS are expected.
+func newNodeAuthCreds(c *cfg, adminCertFile, adminKeyFile string) credentials.TransportCredentials {
+	return &nodeAuthCreds{
+		tls:   credentials.NewTLS(serverTLSConfig(c, adminCertFile, adminKeyFile)),
+		plain: insecure.NewCredentials(),
+	}
+}
+
+func (a *nodeAuthCreds) ServerHandshake(raw net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	first := make([]byte, 1)
+	if _, err := io.ReadFull(raw, first); err != nil {
+		return nil, nil, fmt.Errorf("peek first connection byte: %w", err)
+	}
+	conn := &prefixConn{Conn: raw, prefix: first}
+	if first[0] == tlsRecordTypeHandshake {
+		return a.tls.ServerHandshake(conn)
+	}
+	return a.plain.ServerHandshake(conn)
+}
+
+func (a *nodeAuthCreds) ClientHandshake(ctx context.Context, authority string, raw net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return a.plain.ClientHandshake(ctx, authority, raw)
+}
+
+func (a *nodeAuthCreds) Info() credentials.ProtocolInfo { return a.tls.Info() }
+
+func (a *nodeAuthCreds) Clone() credentials.TransportCredentials {
+	return &nodeAuthCreds{tls: a.tls.Clone(), plain: a.plain.Clone()}
+}
+
+func (a *nodeAuthCreds) OverrideServerName(string) error { return nil }
+
+// prefixConn is a net.Conn that replays a number of already-read bytes before
+// continuing with the underlying connection. It lets the handshake sniffer put
+// the peeked byte back so the chosen handshaker sees the full stream.
+type prefixConn struct {
+	net.Conn
+	prefix []byte
+}
+
+func (c *prefixConn) Read(p []byte) (int, error) {
+	if len(c.prefix) > 0 {
+		n := copy(p, c.prefix)
+		c.prefix = c.prefix[n:]
+		return n, nil
+	}
+	return c.Conn.Read(p)
+}
+
+func serverTLSConfig(c *cfg, adminCertFile, adminKeyFile string) *tls.Config {
+	node := nodeServerTLSConfig(c)
+	return &tls.Config{
+		MinVersion: tls.VersionTLS12,
+		NextProtos: []string{"h2"},
+		GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
+			if hello.ServerName == peerauth.TLSServerName {
+				return node, nil
+			}
+			if adminKeyFile == "" {
+				return nil, fmt.Errorf("no server TLS certificate configured for client %q", hello.ServerName)
+			}
+			cert, err := tls.LoadX509KeyPair(adminCertFile, adminKeyFile)
+			if err != nil {
+				return nil, fmt.Errorf("reload TLS certificate: %w", err)
+			}
+			return &tls.Config{
+				Certificates: []tls.Certificate{cert},
+				MinVersion:   tls.VersionTLS12,
+				NextProtos:   []string{"h2"},
+			}, nil
+		},
+	}
+}
+
+// nodeServerTLSConfig builds the TLS sub-configuration used for inter-node mTLS
+// connections. Only fellow storage nodes are expected here: the verifier
+// rejects any peer whose key is not in the current network map.
+func nodeServerTLSConfig(c *cfg) *tls.Config {
+	return &tls.Config{
+		Certificates:          []tls.Certificate{c.tlsCert},
+		ClientAuth:            tls.RequireAnyClientCert,
+		VerifyPeerCertificate: makeNetmapVerifier(c),
+		MinVersion:            tls.VersionTLS12,
+		NextProtos:            []string{"h2"},
+	}
+}
+
+// makeNetmapVerifier returns a TLS VerifyPeerCertificate hook that accepts a
+// client certificate only if its public key belongs to a node in the current
+// network map.
+func makeNetmapVerifier(c *cfg) func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
+	return func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
+		pub, err := peerauth.PeerPubKey(rawCerts)
+		if err != nil {
+			return fmt.Errorf("extract peer pubkey: %w", err)
+		}
+
+		if !isNetmapNode(c, pub) {
+			c.log.Warn("mTLS: rejecting inter-node peer absent from network map",
+				zap.String("pubkey", hex.EncodeToString(pub)))
+			return fmt.Errorf("peer key %s is not a network map node", hex.EncodeToString(pub))
+		}
+
+		c.log.Info("mTLS: peer authenticated as known SN",
+			zap.String("pubkey", hex.EncodeToString(pub)))
+		return nil
+	}
+}
+
+// isNetmapNode reports whether pub is the public key of some node in the
+// current network map snapshot.
+func isNetmapNode(c *cfg, pub []byte) bool {
+	val := c.netMap.Load()
+	if val == nil {
+		return false
+	}
+	nm, ok := val.(netmap.NetMap)
+	if !ok {
+		return false
+	}
+	for _, n := range nm.Nodes() {
+		if bytes.Equal(n.PublicKey(), pub) {
+			return true
+		}
+	}
+	return false
+}