From 3bc7c6ea8d194e1c1c39f6fe3cac84e7caf1ad63 Mon Sep 17 00:00:00 2001 From: Marat Abrarov Date: Wed, 14 May 2025 20:34:04 +0300 Subject: [PATCH 1/6] fix(kafka): waiting for mapped port without trying to connect to it (which is not possible due to Kafka is not started at that point of time) in container post start hook which prepares configuration for Kafka (fix for https://github.com/testcontainers/testcontainers-go/issues/2748). Signed-off-by: Marat Abrarov --- modules/kafka/kafka.go | 5 +-- wait/host_port.go | 40 +++++++++++++++----- wait/host_port_test.go | 84 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 116 insertions(+), 13 deletions(-) diff --git a/modules/kafka/kafka.go b/modules/kafka/kafka.go index 27fb2d55cb..384a1d3898 100644 --- a/modules/kafka/kafka.go +++ b/modules/kafka/kafka.go @@ -123,10 +123,9 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom // copyStarterScript copies the starter script into the container. func copyStarterScript(ctx context.Context, c testcontainers.Container) error { - if err := wait.ForListeningPort(publicPort). - SkipInternalCheck(). + if err := wait.ForMappedPort(publicPort). WaitUntilReady(ctx, c); err != nil { - return fmt.Errorf("wait for exposed port: %w", err) + return fmt.Errorf("wait for mapped port: %w", err) } host, err := c.Host(ctx) diff --git a/wait/host_port.go b/wait/host_port.go index 5b9a08d55a..14e1ad97ec 100644 --- a/wait/host_port.go +++ b/wait/host_port.go @@ -42,6 +42,11 @@ type HostPortStrategy struct { // a shell is not available in the container or when the container doesn't bind // the port internally until additional conditions are met. skipInternalCheck bool + + // skipExternalCheck is a flag to skip the external check, which, if used with + // skipInternalCheck, makes strategy waiting only for port mapping completion + // without accessing port. + skipExternalCheck bool } // NewHostPortStrategy constructs a default host port strategy that waits for the given @@ -70,6 +75,12 @@ func ForExposedPort() *HostPortStrategy { return NewHostPortStrategy("") } +// ForMappedPort returns a host port strategy that waits for the given port +// to be mapped without accessing the port itself. +func ForMappedPort(port nat.Port) *HostPortStrategy { + return NewHostPortStrategy(port).SkipInternalCheck().SkipExternalCheck() +} + // SkipInternalCheck changes the host port strategy to skip the internal check, // which is useful when a shell is not available in the container or when the // container doesn't bind the port internally until additional conditions are met. @@ -79,6 +90,15 @@ func (hp *HostPortStrategy) SkipInternalCheck() *HostPortStrategy { return hp } +// SkipExternalCheck changes the host port strategy to skip the external check, +// which, if used with SkipInternalCheck, makes strategy waiting only for port +// mapping completion without accessing port. +func (hp *HostPortStrategy) SkipExternalCheck() *HostPortStrategy { + hp.skipExternalCheck = true + + return hp +} + // WithStartupTimeout can be used to change the default startup timeout func (hp *HostPortStrategy) WithStartupTimeout(startupTimeout time.Duration) *HostPortStrategy { hp.timeout = &startupTimeout @@ -124,16 +144,12 @@ func (hp *HostPortStrategy) WaitUntilReady(ctx context.Context, target StrategyT ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - ipAddress, err := target.Host(ctx) - if err != nil { - return err - } - waitInterval := hp.PollInterval internalPort := hp.Port i := 0 if internalPort == "" { + var err error // Port is not specified, so we need to detect it. internalPort, err = hp.detectInternalPort(ctx, target) if err != nil { @@ -157,8 +173,7 @@ func (hp *HostPortStrategy) WaitUntilReady(ctx context.Context, target StrategyT } } - var port nat.Port - port, err = target.MappedPort(ctx, internalPort) + port, err := target.MappedPort(ctx, internalPort) i = 0 for port == "" { @@ -178,8 +193,15 @@ func (hp *HostPortStrategy) WaitUntilReady(ctx context.Context, target StrategyT } } - if err := externalCheck(ctx, ipAddress, port, target, waitInterval); err != nil { - return fmt.Errorf("external check: %w", err) + if !hp.skipExternalCheck { + ipAddress, err := target.Host(ctx) + if err != nil { + return fmt.Errorf("host: %w", err) + } + + if err := externalCheck(ctx, ipAddress, port, target, waitInterval); err != nil { + return fmt.Errorf("external check: %w", err) + } } if hp.skipInternalCheck { diff --git a/wait/host_port_test.go b/wait/host_port_test.go index fdf16da64d..cd89d21918 100644 --- a/wait/host_port_test.go +++ b/wait/host_port_test.go @@ -61,7 +61,89 @@ func TestWaitForListeningPortSucceeds(t *testing.T) { require.NoError(t, err) } -func TestWaitForExposedPortSucceeds(t *testing.T) { +func TestWaitForListeningPortInternallySucceeds(t *testing.T) { + localPort, err := nat.NewPort("tcp", "80") + require.NoError(t, err) + + mappedPort, err := nat.NewPort("tcp", "8080") + require.NoError(t, err) + + var mappedPortCount, execCount int + target := &MockStrategyTarget{ + HostImpl: func(_ context.Context) (string, error) { + return "localhost", nil + }, + MappedPortImpl: func(_ context.Context, p nat.Port) (nat.Port, error) { + if p.Int() != localPort.Int() { + return "", ErrPortNotFound + } + defer func() { mappedPortCount++ }() + if mappedPortCount <= 2 { + return "", ErrPortNotFound + } + return mappedPort, nil + }, + StateImpl: func(_ context.Context) (*container.State, error) { + return &container.State{ + Running: true, + }, nil + }, + ExecImpl: func(_ context.Context, _ []string, _ ...exec.ProcessOption) (int, io.Reader, error) { + defer func() { execCount++ }() + if execCount <= 2 { + return 1, nil, nil + } + return 0, nil, nil + }, + } + + wg := ForListeningPort(localPort). + SkipExternalCheck(). + WithStartupTimeout(5 * time.Second). + WithPollInterval(100 * time.Millisecond) + + err = wg.WaitUntilReady(context.Background(), target) + require.NoError(t, err) +} + +func TestWaitForMappedPortSucceeds(t *testing.T) { + localPort, err := nat.NewPort("tcp", "80") + require.NoError(t, err) + + mappedPort, err := nat.NewPort("tcp", "8080") + require.NoError(t, err) + + var mappedPortCount int + target := &MockStrategyTarget{ + HostImpl: func(_ context.Context) (string, error) { + return "localhost", nil + }, + MappedPortImpl: func(_ context.Context, p nat.Port) (nat.Port, error) { + if p.Int() != localPort.Int() { + return "", ErrPortNotFound + } + defer func() { mappedPortCount++ }() + if mappedPortCount <= 2 { + return "", ErrPortNotFound + } + return mappedPort, nil + }, + StateImpl: func(_ context.Context) (*container.State, error) { + return &container.State{ + Running: true, + }, nil + }, + } + + wg := ForMappedPort(localPort). + WithStartupTimeout(5 * time.Second). + WithPollInterval(100 * time.Millisecond) + + err = wg.WaitUntilReady(context.Background(), target) + require.NoError(t, err) +} + +func TestWaitForExposedPortSkipChecksSucceeds(t *testing.T) { listener, err := net.Listen("tcp", "localhost:0") require.NoError(t, err) defer listener.Close() From 1914d089ca261c8844988f5d82f14188526d9291 Mon Sep 17 00:00:00 2001 From: Marat Abrarov Date: Fri, 16 May 2025 00:33:05 +0300 Subject: [PATCH 2/6] fix(redpanda): waiting for mapped ports without trying to connect to them (which is not possible due to Redpanda is not started at that point of time) in container post start hook before preparing configuration for Redpanda which requires completion of port mapping (fix for https://github.com/testcontainers/testcontainers-go/issues/2748). Signed-off-by: Marat Abrarov --- modules/redpanda/redpanda.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/redpanda/redpanda.go b/modules/redpanda/redpanda.go index 5620a60666..3cfc4ed9b1 100644 --- a/modules/redpanda/redpanda.go +++ b/modules/redpanda/redpanda.go @@ -87,11 +87,12 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom "--memory=1G", }, WaitingFor: wait.ForAll( - // Wait for the ports to be exposed only as the container needs configuration - // before it will bind to the ports and be ready to serve requests. - wait.ForListeningPort(defaultKafkaAPIPort).SkipInternalCheck(), - wait.ForListeningPort(defaultAdminAPIPort).SkipInternalCheck(), - wait.ForListeningPort(defaultSchemaRegistryPort).SkipInternalCheck(), + // Wait for the ports to be mapped without accessing them, + // because container needs Redpanda configuration before Redpanda is started + // and the mapped ports are part of that configuration. + wait.ForMappedPort(defaultKafkaAPIPort), + wait.ForMappedPort(defaultAdminAPIPort), + wait.ForMappedPort(defaultSchemaRegistryPort), ), }, Started: true, From dc2b51f9f04a52ff10efe004ddf34361d45afa68 Mon Sep 17 00:00:00 2001 From: Marat Abrarov Date: Fri, 16 May 2025 00:56:25 +0300 Subject: [PATCH 3/6] fix(redpanda): UNIX path for target file when copying files into container with Redpanda (which is Linux based) to run correctly when host OS is Windows. Signed-off-by: Marat Abrarov --- modules/redpanda/redpanda.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/redpanda/redpanda.go b/modules/redpanda/redpanda.go index 3cfc4ed9b1..7a21233dd3 100644 --- a/modules/redpanda/redpanda.go +++ b/modules/redpanda/redpanda.go @@ -10,7 +10,7 @@ import ( "fmt" "math" "net/http" - "path/filepath" + "path" "strings" "text/template" "time" @@ -159,7 +159,7 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom }, testcontainers.ContainerFile{ Reader: bytes.NewReader(bootstrapConfig), - ContainerFilePath: filepath.Join(redpandaDir, bootstrapConfigFile), + ContainerFilePath: path.Join(redpandaDir, bootstrapConfigFile), FileMode: 600, }, ) @@ -169,12 +169,12 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom req.Files = append(req.Files, testcontainers.ContainerFile{ Reader: bytes.NewReader(settings.cert), - ContainerFilePath: filepath.Join(redpandaDir, certFile), + ContainerFilePath: path.Join(redpandaDir, certFile), FileMode: 600, }, testcontainers.ContainerFile{ Reader: bytes.NewReader(settings.key), - ContainerFilePath: filepath.Join(redpandaDir, keyFile), + ContainerFilePath: path.Join(redpandaDir, keyFile), FileMode: 600, }, ) @@ -207,7 +207,7 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom return c, err } - err = ctr.CopyToContainer(ctx, nodeConfig, filepath.Join(redpandaDir, "redpanda.yaml"), 0o600) + err = ctr.CopyToContainer(ctx, nodeConfig, path.Join(redpandaDir, "redpanda.yaml"), 0o600) if err != nil { return c, fmt.Errorf("copy to container: %w", err) } From 02f2a0ffb259816606c2580d5a074a3ab61ea38c Mon Sep 17 00:00:00 2001 From: Marat Abrarov Date: Fri, 16 May 2025 01:12:26 +0300 Subject: [PATCH 4/6] fix(redpanda): support of remote Docker Engine in tests utilizing TLS. Signed-off-by: Marat Abrarov --- modules/redpanda/go.mod | 2 +- modules/redpanda/go.sum | 4 +- modules/redpanda/redpanda_test.go | 72 +++++++++++++++++++++---------- 3 files changed, 53 insertions(+), 25 deletions(-) diff --git a/modules/redpanda/go.mod b/modules/redpanda/go.mod index 74d94e31f3..c17fc8beec 100644 --- a/modules/redpanda/go.mod +++ b/modules/redpanda/go.mod @@ -47,7 +47,7 @@ require ( github.com/klauspost/compress v1.18.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect - github.com/mdelapenya/tlscert v0.1.0 + github.com/mdelapenya/tlscert v0.2.0 github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect diff --git a/modules/redpanda/go.sum b/modules/redpanda/go.sum index b227005581..0f43b7c986 100644 --- a/modules/redpanda/go.sum +++ b/modules/redpanda/go.sum @@ -59,8 +59,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mdelapenya/tlscert v0.1.0 h1:YTpF579PYUX475eOL+6zyEO3ngLTOUWck78NBuJVXaM= -github.com/mdelapenya/tlscert v0.1.0/go.mod h1:wrbyM/DwbFCeCeqdPX/8c6hNOqQgbf0rUDErE1uD+64= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= diff --git a/modules/redpanda/redpanda_test.go b/modules/redpanda/redpanda_test.go index 3addcb69ea..a315c67f95 100644 --- a/modules/redpanda/redpanda_test.go +++ b/modules/redpanda/redpanda_test.go @@ -20,14 +20,17 @@ import ( "github.com/twmb/franz-go/pkg/sasl/scram" "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/log" "github.com/testcontainers/testcontainers-go/modules/redpanda" "github.com/testcontainers/testcontainers-go/network" ) +const testImage = "docker.redpanda.com/redpandadata/redpanda:v23.3.3" + func TestRedpanda(t *testing.T) { ctx := context.Background() - ctr, err := redpanda.Run(ctx, "docker.redpanda.com/redpandadata/redpanda:v23.3.3") + ctr, err := redpanda.Run(ctx, testImage) testcontainers.CleanupContainer(t, ctr) require.NoError(t, err) @@ -78,7 +81,7 @@ func TestRedpandaWithAuthentication(t *testing.T) { ctx := context.Background() // redpandaCreateContainer { ctr, err := redpanda.Run(ctx, - "docker.redpanda.com/redpandadata/redpanda:v23.3.3", + testImage, redpanda.WithEnableSASL(), redpanda.WithEnableKafkaAuthorization(), redpanda.WithEnableWasmTransform(), @@ -192,7 +195,7 @@ func TestRedpandaWithAuthentication(t *testing.T) { func TestRedpandaWithBootstrapUserAuthentication(t *testing.T) { ctx := context.Background() ctr, err := redpanda.Run(ctx, - "docker.redpanda.com/redpandadata/redpanda:v23.3.3", + testImage, redpanda.WithEnableSASL(), redpanda.WithEnableKafkaAuthorization(), redpanda.WithEnableWasmTransform(), @@ -427,7 +430,7 @@ func TestRedpandaWithOldVersionAndWasm(t *testing.T) { func TestRedpandaProduceWithAutoCreateTopics(t *testing.T) { ctx := context.Background() - ctr, err := redpanda.Run(ctx, "docker.redpanda.com/redpandadata/redpanda:v23.3.3", redpanda.WithAutoCreateTopics()) + ctr, err := redpanda.Run(ctx, testImage, redpanda.WithAutoCreateTopics()) testcontainers.CleanupContainer(t, ctr) require.NoError(t, err) @@ -446,17 +449,17 @@ func TestRedpandaProduceWithAutoCreateTopics(t *testing.T) { } func TestRedpandaWithTLS(t *testing.T) { - tmp := t.TempDir() - cert := tlscert.SelfSignedFromRequest(tlscert.Request{ - Name: "client", - Host: "localhost,127.0.0.1", - ParentDir: tmp, - }) - require.NotNil(t, cert, "failed to generate cert") - ctx := context.Background() - ctr, err := redpanda.Run(ctx, "docker.redpanda.com/redpandadata/redpanda:v23.3.3", redpanda.WithTLS(cert.Bytes, cert.KeyBytes)) + containerHostAddress, err := containerHost(ctx) + require.NoError(t, err) + cert, err := tlscert.SelfSignedFromRequestE(tlscert.Request{ + Name: "client", + Host: "localhost,127.0.0.1," + containerHostAddress, + }) + require.NoError(t, err, "failed to generate cert") + + ctr, err := redpanda.Run(ctx, testImage, redpanda.WithTLS(cert.Bytes, cert.KeyBytes)) testcontainers.CleanupContainer(t, ctr) require.NoError(t, err) @@ -509,19 +512,18 @@ func TestRedpandaWithTLS(t *testing.T) { } func TestRedpandaWithTLSAndSASL(t *testing.T) { - tmp := t.TempDir() + ctx := context.Background() - cert := tlscert.SelfSignedFromRequest(tlscert.Request{ - Name: "client", - Host: "localhost,127.0.0.1", - ParentDir: tmp, + containerHostAddress, err := containerHost(ctx) + require.NoError(t, err) + cert, err := tlscert.SelfSignedFromRequestE(tlscert.Request{ + Name: "client", + Host: "localhost,127.0.0.1," + containerHostAddress, }) - require.NotNil(t, cert, "failed to generate cert") - - ctx := context.Background() + require.NoError(t, err, "failed to generate cert") ctr, err := redpanda.Run(ctx, - "docker.redpanda.com/redpandadata/redpanda:v23.3.3", + testImage, redpanda.WithTLS(cert.Bytes, cert.KeyBytes), redpanda.WithEnableSASL(), redpanda.WithEnableKafkaAuthorization(), @@ -698,3 +700,29 @@ func TestRedpandaBootstrapConfig(t *testing.T) { require.False(t, needsRestart) } } + +func containerHost(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (string, error) { + // Use a dummy request to get the provider from options. + var req testcontainers.GenericContainerRequest + for _, opt := range opts { + if err := opt.Customize(&req); err != nil { + return "", err + } + } + + logging := req.Logger + if logging == nil { + logging = log.Default() + } + p, err := req.ProviderType.GetProvider(testcontainers.WithLogger(logging)) + if err != nil { + return "", err + } + + if p, ok := p.(*testcontainers.DockerProvider); ok { + return p.DaemonHost(ctx) + } + + // Fall back to localhost. + return "localhost", nil +} From f5ab3e21b6c2b96d7caea59844af319fb9a7cf91 Mon Sep 17 00:00:00 2001 From: Marat Abrarov Date: Tue, 20 May 2025 19:48:58 +0300 Subject: [PATCH 5/6] chore(redpanda): unused const removed. Signed-off-by: Marat Abrarov --- modules/redpanda/redpanda.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/redpanda/redpanda.go b/modules/redpanda/redpanda.go index 7a21233dd3..37f875bc40 100644 --- a/modules/redpanda/redpanda.go +++ b/modules/redpanda/redpanda.go @@ -38,7 +38,6 @@ const ( defaultKafkaAPIPort = "9092/tcp" defaultAdminAPIPort = "9644/tcp" defaultSchemaRegistryPort = "8081/tcp" - defaultDockerKafkaAPIPort = "29092" redpandaDir = "/etc/redpanda" entrypointFile = "/entrypoint-tc.sh" From f2c854975baa2cad86a4a6cfba50d8712859debc Mon Sep 17 00:00:00 2001 From: Marat Abrarov Date: Fri, 23 May 2025 20:42:10 +0300 Subject: [PATCH 6/6] docs(wait): skipping external check when waiting for listening port, waiting for port mapping completion. Signed-off-by: Marat Abrarov --- docs/features/wait/host_port.md | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/features/wait/host_port.md b/docs/features/wait/host_port.md index 10531e5e64..c96b4ce679 100644 --- a/docs/features/wait/host_port.md +++ b/docs/features/wait/host_port.md @@ -60,3 +60,42 @@ req := ContainerRequest{ WaitingFor: wait.ForExposedPort().SkipInternalCheck(), } ``` + +## Skipping the external check + +_Testcontainers for Go_ checks if the container is listening to the port externally (outside of container, +from the host where _Testcontainers for Go_ is used) before returning the control to the caller. + +But there are cases where this external check is not needed. +In this case, the `wait.ForListeningPort.SkipExternalCheck` can be used to skip the external check. + +```golang +req := ContainerRequest{ + Image: "nginx:alpine", + // Do not check port 80 externally, check it internally only + WaitingFor: wait.ForListeningPort("80/tcp").SkipExternalCheck(), +} +``` + +If there is a need to wait only for completion of container port mapping (which doesn't happen immediately after container is started), +then both internal and external checks can be skipped: + +```golang +req := ContainerRequest{ + Image: "nginx:alpine", + ExposedPorts: []string{"80/tcp"}, + // Wait only for completion of port 80 mapping (from container runtime perspective), do not connect to 80 port + WaitingFor: wait.ForListeningPort("80/tcp").SkipInternalCheck().SkipExternalCheck(), +} +``` + +Alternatively, `wait.ForMappedPort` can be used: + +```golang +req := ContainerRequest{ + Image: "nginx:alpine", + ExposedPorts: []string{"80/tcp"}, + // Wait only for completion of port 80 mapping (from container runtime perspective), do not connect to 80 port + WaitingFor: wait.ForMappedPort("80/tcp"), +} +```