Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/usages/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ At least one join or Azure authentication method must be configured. `azure.boot
| `azure.servicePrincipal.tenantId` | string | Microsoft Entra tenant ID for the service principal. | `70a036f6-8e4d-4615-bad6-149c02e7720d` |
| `azure.servicePrincipal.clientId` | string | Application client ID. | `00000000-0000-0000-0000-000000000000` |
| `azure.servicePrincipal.clientSecret` | string | Application client secret. Mutually exclusive with `clientSecretFile`. | `<client-secret>` |
| `azure.servicePrincipal.clientSecretFile` | string | Path to a protected regular file (no group/other access; e.g., 0600) containing the application client secret. Mutually exclusive with `clientSecret`. | `/run/credentials/aks-flex-node-sp` |
| `azure.servicePrincipal.clientSecretFile` | string | Path to a protected regular file (no group/other access; e.g., 0600) containing either the application client secret or a PEM/unencrypted PFX certificate and private key. PFX certificate files must use a `.pfx` suffix. | `/run/credentials/aks-flex-node-sp` |

## Agent

Expand Down
2 changes: 1 addition & 1 deletion docs/usages/joining-nodes.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ Minimal config shape:
}
```

The credential file contains only the client secret. The agent requires this to be a non-empty regular file with no group/world access (for example, mode 0600). Inline `clientSecret` remains supported, but cannot be configured together with `clientSecretFile`.
The credential file contains either the client secret or a PEM/unencrypted PFX application certificate and private key; the agent detects the credential type from its contents. PFX certificate files must use a `.pfx` suffix. The file must be a non-empty regular file with no group/world access (for example, mode 0600). Only one of `clientSecret` or `clientSecretFile` can be configured.

## Authentication Mode Selection

Expand Down
13 changes: 13 additions & 0 deletions pkg/aksmachine/client_armapi.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,19 @@ func getCredential(cfg *config.Config, logger *slog.Logger, clientOpts azcore.Cl
"tenantID", cfg.Azure.ServicePrincipal.TenantID,
"clientID", cfg.Azure.ServicePrincipal.ClientID,
)
if cfg.Azure.ServicePrincipal.ClientSecretFile != "" {
certificates, privateKey, err := cfg.Azure.ServicePrincipal.LoadClientCertificate()
if err != nil {
return nil, fmt.Errorf("load service principal client certificate: %w", err)
}
return azidentity.NewClientCertificateCredential(
cfg.Azure.ServicePrincipal.TenantID,
cfg.Azure.ServicePrincipal.ClientID,
certificates,
privateKey,
&azidentity.ClientCertificateCredentialOptions{ClientOptions: clientOpts},
)
}
return azidentity.NewClientSecretCredential(
cfg.Azure.ServicePrincipal.TenantID,
cfg.Azure.ServicePrincipal.ClientID,
Expand Down
26 changes: 26 additions & 0 deletions pkg/aksmachine/client_armapi_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
package aksmachine

import (
"io"
"log/slog"
"math"
"path/filepath"
"strings"
"testing"

Expand Down Expand Up @@ -150,6 +153,7 @@ func TestAzureClientOptionsFromConfig(t *testing.T) {
if !ok {
t.Fatal("ResourceManager cloud service is missing")
}

if service.Endpoint != "https://management.example.test" {
t.Fatalf("ResourceManager endpoint = %q, want https://management.example.test", service.Endpoint)
}
Expand All @@ -161,6 +165,28 @@ func TestAzureClientOptionsFromConfig(t *testing.T) {
}
}

func TestGetCredentialClientCertificateLoadError(t *testing.T) {
t.Parallel()

cfg := testARMConfig(testClusterResourceID, "flex-node-1", "1.34.0")
cfg.Azure.ServicePrincipal = &config.ServicePrincipalConfig{
TenantID: "tenant",
ClientID: "client",
ClientSecretFile: filepath.Join(t.TempDir(), "missing"),
}
credential, err := getCredential(
cfg,
slog.New(slog.NewTextHandler(io.Discard, nil)),
azureClientOptionsFromConfig(cfg),
)
if err == nil || !strings.Contains(err.Error(), "load service principal client certificate") {
t.Fatalf("getCredential() error = %v, want client certificate load error", err)
}
if credential != nil {
t.Fatalf("getCredential() = %T, want nil", credential)
}
}

func TestBuildK8sProfile(t *testing.T) {
t.Parallel()

Expand Down
19 changes: 19 additions & 0 deletions pkg/cmd/token/kubelogin/kubelogin.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io"
"os"

"github.com/Azure/AKSFlexNode/pkg/config"
"github.com/Azure/kubelogin/pkg/token"
"github.com/spf13/cobra"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -24,6 +25,7 @@ const aksAADServerID = "6dae42f8-4368-4678-94ff-3960e28e3630"
var flagServerID string
var flagPopEnabled bool
var flagPopClaims string
var flagClientCertificateFile string

var Command = &cobra.Command{
Use: "kubelogin",
Expand All @@ -47,6 +49,10 @@ func init() {
&flagPopClaims, "pop-claims", "",
"Comma-separated list of key=value claims to include in the PoP token (e.g., 'u=cluster-resource-id').",
)
Command.Flags().StringVar(
&flagClientCertificateFile, "client-certificate-file", "",
"Path to the service principal client certificate file.",
)
}

func run(ctx context.Context, out io.Writer) error {
Expand All @@ -63,6 +69,12 @@ func run(ctx context.Context, out io.Writer) error {
tokOpts.ServerID = flagServerID
tokOpts.IsPoPTokenEnabled = flagPopEnabled
tokOpts.PoPTokenClaims = flagPopClaims
if flagClientCertificateFile != "" {
if err := validateClientCertificateFile(flagClientCertificateFile); err != nil {
return err
}
tokOpts.ClientCert = flagClientCertificateFile
}
// TODO: logging to show login details
provider, err := token.GetTokenProvider(tokOpts)
if err != nil {
Expand All @@ -76,6 +88,13 @@ func run(ctx context.Context, out io.Writer) error {
return outputToken(out, ec, accessToken)
}

func validateClientCertificateFile(certificateFile string) error {
if err := config.ValidateServicePrincipalCertificateFile(certificateFile); err != nil {
return fmt.Errorf("validate client certificate file: %w", err)
}
return nil
}
Comment on lines +91 to +96

const execInfoEnv = "KUBERNETES_EXEC_INFO"

var scheme = runtime.NewScheme()
Expand Down
98 changes: 98 additions & 0 deletions pkg/cmd/token/kubelogin/kubelogin_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package kubelogin

import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"os"
"path/filepath"
"strings"
"testing"
"time"
)

func TestValidateClientCertificateFile(t *testing.T) {
t.Parallel()

dir := t.TempDir()
validFile := filepath.Join(dir, "client-certificate.pem")
writeTestClientCertificate(t, validFile)
insecureFile := filepath.Join(dir, "client-certificate-insecure.pem")
writeTestClientCertificate(t, insecureFile)
if err := os.Chmod(insecureFile, 0o644); err != nil {
t.Fatalf("os.Chmod: %v", err)
}
invalidFile := filepath.Join(dir, "client-certificate-invalid.pem")
if err := os.WriteFile(invalidFile, []byte("not-a-certificate"), 0o600); err != nil {
t.Fatalf("os.WriteFile: %v", err)
}

tests := []struct {
name string
path string
wantErr string
}{
{name: "valid protected certificate file", path: validFile},
{
name: "rejects insecure permissions",
path: insecureFile,
wantErr: "must not be accessible by group or other users",
},
{
name: "rejects malformed certificate file contents",
path: invalidFile,
wantErr: "parse service principal client certificate file",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()

err := validateClientCertificateFile(tt.path)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("validateClientCertificateFile() error = %v, want %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatalf("validateClientCertificateFile() error = %v", err)
}
})
}
}

func writeTestClientCertificate(t *testing.T, path string) {
t.Helper()

privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("rsa.GenerateKey: %v", err)
}
template := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: "test-client"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
}
certificate, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
if err != nil {
t.Fatalf("x509.CreateCertificate: %v", err)
}
privateKeyData, err := x509.MarshalPKCS8PrivateKey(privateKey)
if err != nil {
t.Fatalf("x509.MarshalPKCS8PrivateKey: %v", err)
}
data := append(
pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate}),
pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyData})...,
)
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatalf("os.WriteFile: %v", err)
}
}
26 changes: 18 additions & 8 deletions pkg/config/adapter.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,21 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig {
ac.Kubelet.Auth.BootstrapToken = cfg.Azure.BootstrapToken.Token

case cfg.IsSPConfigured():
ac.Kubelet.Auth.ExecCredential = buildExecCredential(map[string]string{
"AAD_LOGIN_METHOD": "spn",
"AAD_SERVICE_PRINCIPAL_CLIENT_ID": cfg.Azure.ServicePrincipal.ClientID,
"AAD_SERVICE_PRINCIPAL_CLIENT_SECRET": cfg.Azure.ServicePrincipal.ClientSecret,
"AZURE_TENANT_ID": cfg.Azure.ServicePrincipal.TenantID,
})
env := map[string]string{
"AAD_LOGIN_METHOD": "spn",
"AAD_SERVICE_PRINCIPAL_CLIENT_ID": cfg.Azure.ServicePrincipal.ClientID,
"AZURE_TENANT_ID": cfg.Azure.ServicePrincipal.TenantID,
}
if cfg.Azure.ServicePrincipal.clientCertificateData() != "" {
ac.Kubelet.Auth.ExecCredential = buildExecCredential(
env,
"--client-certificate-file",
cfg.Azure.ServicePrincipal.ClientSecretFile,
)
Comment on lines +82 to +87
} else {
env["AAD_SERVICE_PRINCIPAL_CLIENT_SECRET"] = cfg.Azure.ServicePrincipal.ClientSecret
ac.Kubelet.Auth.ExecCredential = buildExecCredential(env)
}

case cfg.IsMIConfigured():
env := map[string]string{
Expand Down Expand Up @@ -114,16 +123,17 @@ func ResolveMachineGoalState(ctx context.Context, log *slog.Logger, cfg *Config,
// buildExecCredential creates an ExecConfig that invokes the aks-flex-node
// binary as a credential plugin. The binary's `token kubelogin` subcommand
// uses kubelogin to obtain an Azure AD token for the AKS API server.
func buildExecCredential(env map[string]string) *clientcmdapi.ExecConfig {
func buildExecCredential(env map[string]string, args ...string) *clientcmdapi.ExecConfig {
execEnv := make([]clientcmdapi.ExecEnvVar, 0, len(env))
for k, v := range env {
execEnv = append(execEnv, clientcmdapi.ExecEnvVar{Name: k, Value: v})
}
execArgs := append([]string{"token", "kubelogin", "--server-id", aksAADServerID}, args...)

return &clientcmdapi.ExecConfig{
APIVersion: "client.authentication.k8s.io/v1",
Command: flexNodeBinaryPath,
Args: []string{"token", "kubelogin", "--server-id", aksAADServerID},
Args: execArgs,
Env: execEnv,
InteractiveMode: clientcmdapi.NeverExecInteractiveMode,
ProvideClusterInfo: false,
Expand Down
36 changes: 36 additions & 0 deletions pkg/config/adapter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package config
import (
"os"
"path/filepath"
"slices"
"testing"

"github.com/Azure/unbounded/pkg/agent/goalstates"
Expand Down Expand Up @@ -141,6 +142,7 @@ func TestToAgentConfig_ServicePrincipalClientSecretFile(t *testing.T) {
if err := os.WriteFile(clientSecretFile, []byte("file-secret\n"), 0o600); err != nil {
t.Fatalf("os.WriteFile: %v", err)
}

cfg := &Config{
Azure: AzureConfig{
ServicePrincipal: &ServicePrincipalConfig{
Expand All @@ -167,6 +169,40 @@ func TestToAgentConfig_ServicePrincipalClientSecretFile(t *testing.T) {
}
}

func TestToAgentConfig_ServicePrincipalCertificateFile(t *testing.T) {
t.Parallel()

certificateFile := filepath.Join(t.TempDir(), "client-certificate.pem")
writeTestClientCertificate(t, certificateFile)
cfg := &Config{
Azure: AzureConfig{
ServicePrincipal: &ServicePrincipalConfig{
TenantID: "tenant-123",
ClientID: "client-456",
ClientSecretFile: certificateFile,
},
},
}
if err := cfg.Azure.ServicePrincipal.validate(); err != nil {
t.Fatalf("ServicePrincipalConfig.validate() error = %v", err)
}

exec := ToAgentConfig(cfg, "kube1").Kubelet.Auth.ExecCredential
if exec == nil {
t.Fatal("ExecCredential should be set for SP auth")
}
envMap := make(map[string]string)
for _, e := range exec.Env {
envMap[e.Name] = e.Value
}
if !slices.Equal(exec.Args, []string{"token", "kubelogin", "--server-id", aksAADServerID, "--client-certificate-file", certificateFile}) {
t.Fatalf("Args=%v, want kubelogin client-certificate-file args", exec.Args)
}
if _, ok := envMap["AAD_SERVICE_PRINCIPAL_CLIENT_SECRET"]; ok {
t.Fatal("AAD_SERVICE_PRINCIPAL_CLIENT_SECRET should not be set for certificate auth")
}
}

func TestToAgentConfig_ManagedIdentity(t *testing.T) {
t.Parallel()

Expand Down
Loading
Loading