diff --git a/.travis.yml b/.travis.yml index 8e951f37..d4a2f405 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,18 +9,20 @@ go: - "1.13.x" env: - - VERBOSE=1 TEST_SUITE=run-tests-local-process ARANGODB=arangodb:3.6 STARTER_MODES="single,cluster,activefailover" - - VERBOSE=1 TEST_SUITE=run-tests-docker ARANGODB=arangodb:3.6 STARTER_MODES="single,cluster,activefailover" - - VERBOSE=1 TEST_SUITE=run-tests-local-process ARANGODB=arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover" - - VERBOSE=1 TEST_SUITE=run-tests-docker ARANGODB=arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover" - - VERBOSE=1 TEST_SUITE=run-tests-local-process ARANGODB=arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" - - VERBOSE=1 TEST_SUITE=run-tests-docker ARANGODB=arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" + - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single" + - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single" + - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover" ON_PR=1 + - VERBOSE=1 TEST_SUITE=run-tests-docker GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover" ON_PR=1 +# - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover" ON_PR=1 +# - VERBOSE=1 TEST_SUITE=run-tests-docker GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover" ON_PR=1 +# - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" ON_PR=1 +# - VERBOSE=1 TEST_SUITE=run-tests-docker GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" ON_PR=1 -script: make $TEST_SUITE +script: 'if ([ "$TRAVIS_PULL_REQUEST" != "false" ] && [ "$ON_PR" != "" ]) || ([ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$ON_PR" == "" ]) ; then env; make $TEST_SUITE; fi' -# Install Docker CE -before_install: - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - sudo apt-get update - - sudo apt-get -y install docker-ce +## Install Docker CE +#before_install: +# - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +# - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +# - sudo apt-get update +# - sudo apt-get -y install docker-ce-cli diff --git a/Dockerfile b/Dockerfile index 650cfcc2..026d056f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM alpine:3.11 +ARG IMAGE=alpine:3.11 +FROM ${IMAGE} MAINTAINER Max Neunhoeffer COPY bin/linux/amd64/arangodb /app/ diff --git a/Makefile b/Makefile index 8f936b58..79db1407 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,8 @@ VERSION_MAJOR := $(shell echo $(VERSION_MAJOR_MINOR) | cut -f 1 -d '.') COMMIT := $(shell git rev-parse --short HEAD) MAKEFILE := $(ROOTDIR)/Makefile +ALPINE_IMAGE ?= alpine:3.11 + DOCKERCLI ?= $(shell which docker) GOBUILDLINKTARGET := ../../../.. @@ -28,7 +30,7 @@ REPOPATH := $(ORGPATH)/$(REPONAME) GOPATH := $(GOBUILDDIR) GOVERSION := 1.13.6 -GOIMAGE := golang:$(GOVERSION) +GOIMAGE ?= golang:$(GOVERSION) GOOS ?= linux GOARCH ?= amd64 @@ -101,6 +103,7 @@ DOCKER_CMD = $(DOCKERCLI) run \ -e GOOS=$(GOOS) \ -e GOARCH=$(GOARCH) \ -e CGO_ENABLED=0 \ + -e TRAVIS=$(TRAVIS) \ $(DOCKER_PARAMS) \ -w /usr/code/ \ $(DOCKER_IMAGE) @@ -149,7 +152,7 @@ $(TESTBIN): $(GOBUILDDIR) $(TEST_SOURCES) $(BIN) $(DOCKER_CMD) go test -c -o "$(TEST_BIN)" ./test docker: build - $(DOCKERCLI) build -t arangodb/arangodb-starter . + $(DOCKERCLI) build -t arangodb/arangodb-starter --build-arg "IMAGE=$(ALPINE_IMAGE)" . docker-push: docker ifneq ($(DOCKERNAMESPACE), arangodb) @@ -196,7 +199,7 @@ run-tests-local-process-run: $(DOCKER_CMD) /usr/code/bin/linux/amd64/test -test.timeout $(TEST_TIMEOUT) -test.v $(TESTOPTIONS) _run-tests: build-test build - @TEST_MODES=$(TEST_MODES) STARTER_MODES=$(STARTER_MODES) STARTER=$(BIN) ENTERPRISE=$(ENTERPRISE) IP=$(IP) ARANGODB=$(ARANGODB) $(TESTBIN) -test.timeout $(TEST_TIMEOUT) -test.v $(TESTOPTIONS) + @TEST_MODES=$(TEST_MODES) STARTER_MODES=$(STARTER_MODES) STARTER=$(BIN) ENTERPRISE=$(ENTERPRISE) IP=$(IP) ARANGODB=$(ARANGODB) $(TESTBIN) -test.timeout $(TEST_TIMEOUT) -test.failfast -test.v $(TESTOPTIONS) ifdef TRAVIS run-tests-docker-pre: docker diff --git a/go.mod b/go.mod index 54a1f3c9..e8282c0d 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ replace golang.org/x/sys => golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a require ( github.com/aktau/github-release v0.8.1 github.com/arangodb-helper/go-certificates v0.0.0-20180821055445-9fca24fc2680 - github.com/arangodb/go-driver v0.0.0-20200624173407-d1c92a8bd2b8 + github.com/arangodb/go-driver v0.0.0-20210608063820-cd7131034c24 github.com/arangodb/go-upgrade-rules v0.0.0-20200605091205-439fb1ee86e7 github.com/cenkalti/backoff v2.2.1+incompatible github.com/coreos/go-semver v0.3.0 @@ -17,7 +17,6 @@ require ( github.com/fatih/color v1.9.0 github.com/fsouza/go-dockerclient v1.6.5 github.com/github-release/github-release v0.8.1 // indirect - github.com/google/addlicense v0.0.0-20200622132530-df58acafd6d5 // indirect github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/kevinburke/rest v0.0.0-20200429221318-0d2892b400f8 // indirect @@ -29,11 +28,9 @@ require ( github.com/ryanuber/columnize v2.1.0+incompatible github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.5.1 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/mod v0.3.0 // indirect - golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 // indirect golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect - golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347 // indirect ) diff --git a/go.sum b/go.sum index 5703a27d..7ce2b599 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/arangodb-helper/go-certificates v0.0.0-20180821055445-9fca24fc2680 h1 github.com/arangodb-helper/go-certificates v0.0.0-20180821055445-9fca24fc2680/go.mod h1:xDyzBwyYzcEhsaDXtmxCNM4p5BrtuoVYYsRTuJqmCeg= github.com/arangodb/go-driver v0.0.0-20200624173407-d1c92a8bd2b8 h1:kWyYp1Gtp4x458LYlk9uAoM5v0ifZX58frROb1pC7Jo= github.com/arangodb/go-driver v0.0.0-20200624173407-d1c92a8bd2b8/go.mod h1:JG79qtPYRxUB6CdGWSH1XwpolSBjthuZX+Iaz/H38rA= +github.com/arangodb/go-driver v0.0.0-20210608063820-cd7131034c24 h1:Tmuxuw6asYMsGa73rB8vGcE4ipX+j1uBFsodYqtCJNE= +github.com/arangodb/go-driver v0.0.0-20210608063820-cd7131034c24/go.mod h1:3NUekcRLpgheFIGEwcOvxilEW73MV1queNKW58k7sdc= github.com/arangodb/go-upgrade-rules v0.0.0-20200605091205-439fb1ee86e7 h1:zCY5fsv5apos+oAdd1bLr1UEFOHeIUDZCItbwU/u6XE= github.com/arangodb/go-upgrade-rules v0.0.0-20200605091205-439fb1ee86e7/go.mod h1:RkPIG6JJ2pcJUoymc18NxAJGraZd+iAEVnOTDjZey/w= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g= @@ -102,11 +104,13 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/addlicense v0.0.0-20200622132530-df58acafd6d5 h1:m6Z1Cm53o4VecQFxKCnvULGfIT0Igo3MX131i+00IIo= github.com/google/addlicense v0.0.0-20200622132530-df58acafd6d5/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= +github.com/google/addlicense v0.0.0-20200817051935-6f4cd4aacc89/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -235,6 +239,7 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -268,6 +273,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -297,6 +303,7 @@ golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347 h1:/e4fNMHdLn7SQSxTrRZTma2xjQW6ELdxcnpqMhpo9X4= golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200818005847-188abfa75333/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/pkg/definitions/server_type.go b/pkg/definitions/server_type.go index 679c914d..d63858f4 100644 --- a/pkg/definitions/server_type.go +++ b/pkg/definitions/server_type.go @@ -24,6 +24,7 @@ package definitions import ( "fmt" + "time" ) // ServerType specifies the types of database servers. @@ -63,6 +64,16 @@ func (s ServerType) PortOffset() int { } } +// InitialStopTimeout returns initial delay for process stopping +func (s ServerType) InitialStopTimeout() time.Duration { + switch s { + case ServerTypeDBServer, ServerTypeSingle, ServerTypeResilientSingle: + return 3 * time.Second + default: + return time.Second + } +} + // ProcessType returns the type of process needed to run a server of given type. func (s ServerType) ProcessType() ProcessType { switch s { diff --git a/service/process_wrapper.go b/service/process_wrapper.go index fbcc8918..86219527 100644 --- a/service/process_wrapper.go +++ b/service/process_wrapper.go @@ -229,7 +229,7 @@ func (p *processWrapper) run(startedCh chan<- struct{}) { case <-p.stopping: if p.s.stopping { // Starter is being closed - terminateProcessWithActions(logProcess, p.proc, p.serverType, 10*time.Second, time.Minute) + terminateProcessWithActions(logProcess, p.proc, p.serverType, time.Second, time.Minute) } else { // Process restart terminateProcessWithActions(logProcess, p.proc, p.serverType, 0, time.Minute) diff --git a/test/docker_activefailover_default_test.go b/test/docker_activefailover_default_test.go index 7c6cb056..2174fe0e 100644 --- a/test/docker_activefailover_default_test.go +++ b/test/docker_activefailover_default_test.go @@ -33,6 +33,9 @@ import ( // TestDockerActiveFailoverDefault runs 3 arangodb starters in docker with mode=activefailover // and otherwise default settings. func TestDockerActiveFailoverDefault(t *testing.T) { + log := GetLogger(t) + defer log.Clean() + needTestMode(t, testModeDocker) needStarterMode(t, starterModeActiveFailover) if os.Getenv("IP") == "" { @@ -48,6 +51,7 @@ func TestDockerActiveFailoverDefault(t *testing.T) { --starter.address=$IP \ --starter.mode=activefailover */ + volID1 := createDockerID("vol-starter-test-activefailover-default1-") createDockerVolume(t, volID1) defer removeDockerVolume(t, volID1) @@ -132,12 +136,9 @@ func TestDockerActiveFailoverDefault(t *testing.T) { testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) } // TestDockerActiveFailover2Instance runs 3 arangodb starters in docker with mode=activefailover @@ -243,10 +244,8 @@ func TestDockerActiveFailover2Instance(t *testing.T) { testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, true) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) } diff --git a/test/docker_activefailover_local_test.go b/test/docker_activefailover_local_test.go index 2b0607e7..cdd17284 100644 --- a/test/docker_activefailover_local_test.go +++ b/test/docker_activefailover_local_test.go @@ -83,10 +83,8 @@ func TestDockerActiveFailoverLocal(t *testing.T) { testResilientSingle(t, insecureStarterEndpoint(0*portIncrement), false, false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement))) } // TestDockerActiveFailoverSecure runs the arangodb starter in docker with mode `activefailover`, @@ -144,8 +142,6 @@ func TestDockerResilientActiveFailoverSecure(t *testing.T) { testResilientSingle(t, secureStarterEndpoint(0*portIncrement), true, false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, secureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(secureStarterEndpoint(0*portIncrement))) } diff --git a/test/docker_cluster_default_test.go b/test/docker_cluster_default_test.go index 0fbcf9e8..c980c9bd 100644 --- a/test/docker_cluster_default_test.go +++ b/test/docker_cluster_default_test.go @@ -127,12 +127,10 @@ func TestDockerClusterDefault(t *testing.T) { testCluster(t, insecureStarterEndpoint(2*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) } // TestOldDockerClusterDefault runs 3 arangodb starters in docker with default settings. @@ -235,7 +233,8 @@ func TestOldDockerClusterDefault(t *testing.T) { if isVerbose { t.Log("Waiting for termination") } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) } diff --git a/test/docker_cluster_diff_logdir_test.go b/test/docker_cluster_diff_logdir_test.go index 9af9744e..265bb6f8 100644 --- a/test/docker_cluster_diff_logdir_test.go +++ b/test/docker_cluster_diff_logdir_test.go @@ -145,12 +145,10 @@ func TestDockerClusterDifferentLogDir(t *testing.T) { testCluster(t, insecureStarterEndpoint(2*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) check := func(rootDir string, expectedFileCount int) { files, err := getRecursiveLogFiles(rootDir) @@ -282,12 +280,10 @@ func TestDockerClusterDifferentLogDirNoLog2File(t *testing.T) { testCluster(t, insecureStarterEndpoint(2*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) check := func(rootDir string, expectedFileCount int) { files, err := getRecursiveLogFiles(rootDir) diff --git a/test/docker_cluster_diff_ports_test.go b/test/docker_cluster_diff_ports_test.go index 4723cc68..7d10c916 100644 --- a/test/docker_cluster_diff_ports_test.go +++ b/test/docker_cluster_diff_ports_test.go @@ -131,10 +131,8 @@ func TestDockerClusterDifferentPorts(t *testing.T) { testCluster(t, "http://localhost:8000", false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, "http://localhost:6000") - ShutdownStarter(t, "http://localhost:7000") - ShutdownStarter(t, "http://localhost:8000") + waitForCallFunction(t, + ShutdownStarterCall("http://localhost:6000"), + ShutdownStarterCall("http://localhost:7000"), + ShutdownStarterCall("http://localhost:8000")) } diff --git a/test/docker_cluster_local_test.go b/test/docker_cluster_local_test.go index 3f2ff1d4..b798a0a6 100644 --- a/test/docker_cluster_local_test.go +++ b/test/docker_cluster_local_test.go @@ -81,10 +81,8 @@ func TestDockerClusterLocal(t *testing.T) { testCluster(t, insecureStarterEndpoint(0*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement))) } // TestDockerClusterLocalAgencySize1 runs the arangodb starter in docker @@ -141,10 +139,8 @@ func TestDockerClusterLocalAgencySize1(t *testing.T) { testCluster(t, insecureStarterEndpoint(0*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement))) } // TestOldDockerClusterLocal runs the arangodb starter in docker with `--local` @@ -197,8 +193,6 @@ func TestOldDockerClusterLocal(t *testing.T) { testCluster(t, insecureStarterEndpoint(0*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement))) } diff --git a/test/docker_cluster_multi_join_test.go b/test/docker_cluster_multi_join_test.go index ea62468e..57f6c947 100644 --- a/test/docker_cluster_multi_join_test.go +++ b/test/docker_cluster_multi_join_test.go @@ -134,10 +134,8 @@ func TestDockerClusterMultipleJoins(t *testing.T) { testCluster(t, "http://localhost:8000", false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, "http://localhost:6000") - ShutdownStarter(t, "http://localhost:7000") - ShutdownStarter(t, "http://localhost:8000") + waitForCallFunction(t, + ShutdownStarterCall("http://localhost:6000"), + ShutdownStarterCall("http://localhost:7000"), + ShutdownStarterCall("http://localhost:8000")) } diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go index 35e8c8be..8e51854c 100644 --- a/test/docker_cluster_recovery_test.go +++ b/test/docker_cluster_recovery_test.go @@ -34,6 +34,10 @@ import ( // TestDockerClusterRecovery starts a master starter in docker, followed by 2 slave starters. // Once started, it destroys one of the starters and attempts a recovery. func TestDockerClusterRecovery(t *testing.T) { + log := GetLogger(t) + + SkipOnTravis(t, "Test does not work on TRAVIS VM") // TODO: Fix needed + needTestMode(t, testModeDocker) needStarterMode(t, starterModeCluster) if os.Getenv("IP") == "" { @@ -71,7 +75,6 @@ func TestDockerClusterRecovery(t *testing.T) { "docker run -i", "--label starter-test=true", "--name=" + cID1, - "--rm", createLicenseKeyOption(), fmt.Sprintf("-p %d:%d", basePort, basePort), fmt.Sprintf("-v %s:/data", volID1), @@ -90,7 +93,6 @@ func TestDockerClusterRecovery(t *testing.T) { "docker run -i", "--label starter-test=true", "--name=" + cID2, - "--rm", createLicenseKeyOption(), fmt.Sprintf("-p %d:%d", basePort+100, basePort+100), fmt.Sprintf("-v %s:/data", volID2), @@ -110,7 +112,6 @@ func TestDockerClusterRecovery(t *testing.T) { "docker run -i", "--label starter-test=true", "--name=" + cID3, - "--rm", createLicenseKeyOption(), fmt.Sprintf("-p %d:%d", basePort+200, basePort+200), fmt.Sprintf("-v %s:/data", volID3), @@ -132,9 +133,7 @@ func TestDockerClusterRecovery(t *testing.T) { testCluster(t, insecureStarterEndpoint(200), false) } - if isVerbose { - t.Log("Start killing slave1 and its servers") - } + log.Log("Kill Server1") // Cluster is up. // Kill starter slave-1 and all its processes @@ -150,15 +149,23 @@ func TestDockerClusterRecovery(t *testing.T) { for _, s := range plist.Servers { containersToKill = append(containersToKill, s.ContainerID) } + + checkpoint := log.Checkpoint() + + checkpoint.Log("Kill docker containers") + killDockerRun2 := Spawn(t, "docker rm -vf "+strings.Join(containersToKill, " ")) killDockerRun2.Wait() + checkpoint.Log("Wait for docker command to stop") + + // Wait for command to close + dockerRun2.Wait() + // Remove entire docker volume removeDockerVolume(t, volID2) - if isVerbose { - t.Log("Starting recovery...") - } + checkpoint.Log("Recovery") // Create new volume recVolID2 := createDockerID("vol-starter-test-cluster-recovery2-recovery-") @@ -171,20 +178,22 @@ func TestDockerClusterRecovery(t *testing.T) { "docker run -i", "--label starter-test=true", "--name=" + cID2 + "recovery-builder", - "--rm", fmt.Sprintf("-v %s:/data", recVolID2), "alpine", fmt.Sprintf("sh -c \"echo %s > /data/RECOVERY\"", recoveryContent), }, " ")) dockerBuildRecoveryRun.Wait() + checkpoint.Log("Wait for port to be closed") + WaitForHttpPortClosed(checkpoint, NewThrottle(time.Second), insecureStarterEndpoint(100)).ExecuteT(t, time.Minute, time.Second) + + checkpoint.Log("Start docker container") // Restart dockerRun2 recCID2 := createDockerID("starter-test-cluster-recovery2-recovery-") recDockerRun2 := Spawn(t, strings.Join([]string{ "docker run -i", "--label starter-test=true", "--name=" + recCID2, - "--rm", createLicenseKeyOption(), fmt.Sprintf("-p %d:%d", basePort+100, basePort+100), fmt.Sprintf("-v %s:/data", recVolID2), @@ -198,6 +207,7 @@ func TestDockerClusterRecovery(t *testing.T) { }, " ")) defer recDockerRun2.Close() defer removeDockerContainer(t, recCID2) + checkpoint.Log("Docker container started") // Wait until recovered if ok := WaitUntilStarterReady(t, whatCluster, 1, recDockerRun2); ok { @@ -212,10 +222,8 @@ func TestDockerClusterRecovery(t *testing.T) { t.Errorf("Expected RECOVERY file to not-exist, got: %s", describe(err)) }*/ - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0)) - ShutdownStarter(t, insecureStarterEndpoint(100)) - ShutdownStarter(t, insecureStarterEndpoint(200)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0)), + ShutdownStarterCall(insecureStarterEndpoint(100)), + ShutdownStarterCall(insecureStarterEndpoint(200))) } diff --git a/test/docker_cluster_sync_test.go b/test/docker_cluster_sync_test.go index 51428c81..659b01db 100644 --- a/test/docker_cluster_sync_test.go +++ b/test/docker_cluster_sync_test.go @@ -149,10 +149,8 @@ func TestDockerClusterSync(t *testing.T) { testClusterWithSync(t, insecureStarterEndpoint(2*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) } diff --git a/test/docker_cluster_upgrade_test.go b/test/docker_cluster_upgrade_test.go index 7aafead9..d0c0e5e5 100644 --- a/test/docker_cluster_upgrade_test.go +++ b/test/docker_cluster_upgrade_test.go @@ -130,10 +130,8 @@ func TestDockerClusterUpgrade(t *testing.T) { testUpgradeProcess(t, insecureStarterEndpoint(0*portIncrement)) - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) } diff --git a/test/docker_database_version_test.go b/test/docker_database_version_test.go index 5648c11c..16da5710 100644 --- a/test/docker_database_version_test.go +++ b/test/docker_database_version_test.go @@ -95,5 +95,6 @@ func TestDockerDatabaseVersion(t *testing.T) { t.Logf("Got database-version %s", v) } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement))) } diff --git a/test/docker_resilientsingle_default_test.go b/test/docker_resilientsingle_default_test.go index b347ebf0..d9ab15b6 100644 --- a/test/docker_resilientsingle_default_test.go +++ b/test/docker_resilientsingle_default_test.go @@ -132,12 +132,10 @@ func TestDockerResilientSingleDefault(t *testing.T) { testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) } // TestDockerResilientSingle2Instance runs 3 arangodb starters in docker with mode=resilientsingle @@ -243,10 +241,8 @@ func TestDockerResilientSingle2Instance(t *testing.T) { testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, true) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) } diff --git a/test/docker_resilientsingle_local_test.go b/test/docker_resilientsingle_local_test.go index 351aff95..ca6e95b1 100644 --- a/test/docker_resilientsingle_local_test.go +++ b/test/docker_resilientsingle_local_test.go @@ -83,10 +83,8 @@ func TestDockerResilientSingleLocal(t *testing.T) { testResilientSingle(t, insecureStarterEndpoint(0*portIncrement), false, false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement))) } // TestDockerResilientSingleLocalSecure runs the arangodb starter in docker with mode `resilientsingle`, @@ -144,8 +142,6 @@ func TestDockerResilientSingleLocalSecure(t *testing.T) { testResilientSingle(t, secureStarterEndpoint(0*portIncrement), true, false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, secureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(secureStarterEndpoint(0*portIncrement))) } diff --git a/test/docker_single_test.go b/test/docker_single_test.go index 5e761cf2..36b5319a 100644 --- a/test/docker_single_test.go +++ b/test/docker_single_test.go @@ -81,10 +81,8 @@ func TestDockerSingle(t *testing.T) { testSingle(t, insecureStarterEndpoint(0*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement))) } // TestDockerSingleAutoKeyFile runs the arangodb starter in docker with `--starter.mode=single` && `--ssl.auto-key` @@ -140,10 +138,8 @@ func TestDockerSingleAutoKeyFile(t *testing.T) { testSingle(t, secureStarterEndpoint(0*portIncrement), true) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, secureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(secureStarterEndpoint(0*portIncrement))) } // TestDockerSingleAutoContainerName runs the arangodb starter in docker with `--starter.mode=single` automatic detection of its container name. @@ -195,10 +191,8 @@ func TestDockerSingleAutoContainerName(t *testing.T) { testSingle(t, insecureStarterEndpoint(0*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement))) } // TestDockerSingleAutoRocksdb runs the arangodb starter in docker with `--server.storage-engine=rocksdb` and a 3.2+ image. @@ -253,10 +247,8 @@ func TestDockerSingleAutoRocksdb(t *testing.T) { testSingle(t, insecureStarterEndpoint(0*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement))) } // TestOldDockerSingleAutoKeyFile runs the arangodb starter in docker with `--mode=single` && `--sslAutoKeyFile` @@ -310,8 +302,6 @@ func TestOldDockerSingleAutoKeyFile(t *testing.T) { testSingle(t, secureStarterEndpoint(0*portIncrement), true) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, secureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(secureStarterEndpoint(0*portIncrement))) } diff --git a/test/docker_util.go b/test/docker_util.go index 7563e5e3..faa1f697 100644 --- a/test/docker_util.go +++ b/test/docker_util.go @@ -29,6 +29,7 @@ import ( "os/exec" "strings" "testing" + "time" ) func createDockerVolume(t *testing.T, id string) { @@ -44,15 +45,39 @@ func removeDockerVolume(t *testing.T, id string) { } func removeDockerContainer(t *testing.T, id string) { + if t.Failed() { + logDockerPS(t) + logDockerLogs(t, id) + } + c := Spawn(t, fmt.Sprintf("docker rm -f -v %s", id)) defer c.Close() c.Wait() } -func stopDockerContainer(t *testing.T, id string) { - c := Spawn(t, fmt.Sprintf("docker stop --time=120 %s", id)) +func logDockerPS(t *testing.T) { + log := GetLogger(t) + + // Dump of logs if failed + c := Spawn(t, fmt.Sprintf("docker ps -a")) + defer c.Close() + + time.Sleep(500 * time.Millisecond) + + c.Wait() + + logProcessOutput(log, c, "List of containers: ") +} + +func logDockerLogs(t *testing.T, id string) { + log := GetLogger(t) + + // Dump of logs if failed + c := Spawn(t, fmt.Sprintf("docker logs --timestamps %s", id)) defer c.Close() c.Wait() + + logProcessOutput(log, c, "Log of container %s: ", id) } func removeDockerContainersByLabel(t *testing.T, labelKeyValue string) { diff --git a/test/gexpect.go b/test/gexpect.go index 15a37b77..f9cbb1f4 100644 --- a/test/gexpect.go +++ b/test/gexpect.go @@ -32,8 +32,11 @@ import ( "regexp" "sync" "syscall" + "testing" "time" + "github.com/stretchr/testify/require" + "github.com/pkg/errors" ) @@ -85,7 +88,6 @@ func (sp *SubProcess) Start() error { for { n, err := rd.Read(byteBuf) sp.writeOutput(byteBuf[:n]) - sp.matchExpressions() if err != nil { break } @@ -129,6 +131,7 @@ func (sp *SubProcess) SendIntr() error { // Kill the process after the given timeout. func (sp *SubProcess) WaitTimeout(timeout time.Duration) error { done := make(chan struct{}) + defer close(done) go func() { select { case <-time.After(timeout): @@ -139,7 +142,15 @@ func (sp *SubProcess) WaitTimeout(timeout time.Duration) error { } }() err := sp.cmd.Wait() - close(done) + + if err != nil { + if c, ok := err.(*os.SyscallError); ok { + if c.Syscall == "waitid" { + return nil + } + } + } + return maskAny(err) } @@ -151,6 +162,25 @@ func (sp *SubProcess) Wait() error { return nil } +// WaitT waits for the process to terminate with require. +func (sp *SubProcess) WaitT(t *testing.T) { + require.NoError(t, sp.Wait()) +} + +// Output get current output +func (sp *SubProcess) Output() []byte { + sp.mutex.Lock() + defer sp.mutex.Unlock() + + d := sp.output.Bytes() + + r := make([]byte, len(d)) + + copy(r, d) + + return r +} + // ExpectTimeout waits for the output of the process to match the given expression, or until a timeout occurs. // If a match on the given expression is found, the process output is discard until the end of the match and // nil is returned, otherwise a timeout error is returned. @@ -158,11 +188,7 @@ func (sp *SubProcess) Wait() error { func (sp *SubProcess) ExpectTimeout(ctx context.Context, timeout time.Duration, re *regexp.Regexp, id string) error { found := make(chan struct{}) - sp.mutex.Lock() - sp.expressions[re] = found - sp.mutex.Unlock() - - sp.matchExpressions() + sp.matchExpressionAsync(ctx, found, re) select { case <-ctx.Done(): @@ -192,21 +218,35 @@ func (sp *SubProcess) writeOutput(data []byte) { sp.output.Write(data) } -func (sp *SubProcess) matchExpressions() { +func (sp *SubProcess) matchExpressionAsync(ctx context.Context, found chan<- struct{}, regexes ...*regexp.Regexp) { + go func() { + defer close(found) + + ticker := time.NewTicker(125 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if sp.matchExpressionInOutput(regexes...) { + return + } + case <-ctx.Done(): + return + } + } + }() +} + +func (sp *SubProcess) matchExpressionInOutput(regexes ...*regexp.Regexp) bool { sp.mutex.Lock() defer sp.mutex.Unlock() - - for re, found := range sp.expressions { - loc := re.FindIndex(sp.output.Bytes()) - if loc == nil { - // No match - continue + data := sp.output.Bytes() + for _, re := range regexes { + if loc := re.FindIndex(data); loc != nil { + return true } - // Found a match, remove everything until the end of the match - n := loc[1] - sp.output.Next(n) - close(found) - // Remove from map - delete(sp.expressions, re) } + + return false } diff --git a/test/log.go b/test/log.go new file mode 100644 index 00000000..89e85948 --- /dev/null +++ b/test/log.go @@ -0,0 +1,120 @@ +// +// DISCLAIMER +// +// Copyright 2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package test + +import ( + "fmt" + "sync" + "testing" + "time" +) + +var ( + loggerMutex sync.Mutex + loggers = map[*testing.T]Logger{} +) + +func cleanLogger(t *testing.T) { + loggerMutex.Lock() + defer loggerMutex.Unlock() + + delete(loggers, t) +} + +func getLogger(parent *logger, t *testing.T) Logger { + loggerMutex.Lock() + defer loggerMutex.Unlock() + + if l, ok := loggers[t]; ok { + return l + } + + l := &logger{ + start: time.Now(), + t: t, + parent: parent, + } + + loggers[t] = l + return l +} + +type Logger interface { + Log(format string, args ...interface{}) + + SubLogger(t *testing.T) Logger + Checkpoint() Logger + + Clean() +} + +type logger struct { + start time.Time + t *testing.T + + parent *logger +} + +func (l *logger) Checkpoint() Logger { + return &logger{ + start: time.Now(), + t: l.t, + parent: l, + } +} + +func (l *logger) Clean() { + cleanLogger(l.t) +} + +func (l *logger) getParent() *logger { + if l == nil || l.parent == nil { + return nil + } + + if p := l.parent.getParent(); p == nil { + return l + } else { + return p + } +} + +func (l *logger) Log(format string, args ...interface{}) { + line := fmt.Sprintf(format, args...) + if p := l.getParent(); p == nil { + line = fmt.Sprintf("Started: %s > %s", time.Now().Sub(l.start), line) + } else { + line = fmt.Sprintf("Started: %s, In Test: %s > %s", time.Now().Sub(p.start).String(), time.Now().Sub(l.start).String(), line) + } + + l.t.Log(line) + println(line) +} + +func (l *logger) SubLogger(t *testing.T) Logger { + return getLogger(l, t) +} + +func GetLogger(t *testing.T) Logger { + return getLogger(nil, t) +} diff --git a/test/process_activefailover_default_test.go b/test/process_activefailover_default_test.go index 7fbb00a8..4410dd84 100644 --- a/test/process_activefailover_default_test.go +++ b/test/process_activefailover_default_test.go @@ -96,10 +96,8 @@ func TestProcessActiveFailoverDefaultShutdownViaAPI(t *testing.T) { testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) } diff --git a/test/process_cluster_default_test.go b/test/process_cluster_default_test.go index d315b58f..fca46258 100644 --- a/test/process_cluster_default_test.go +++ b/test/process_cluster_default_test.go @@ -94,12 +94,10 @@ func TestProcessClusterDefaultShutdownViaAPI(t *testing.T) { testCluster(t, insecureStarterEndpoint(2*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) } // TestOldProcessClusterDefault starts a master starter, followed by 2 slave starters. diff --git a/test/process_cluster_local_test.go b/test/process_cluster_local_test.go index 3b6be68a..793344d9 100644 --- a/test/process_cluster_local_test.go +++ b/test/process_cluster_local_test.go @@ -74,10 +74,8 @@ func TestProcessClusterLocalShutdownViaAPI(t *testing.T) { testCluster(t, insecureStarterEndpoint(2*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement))) } // TestOldProcessClusterLocal runs `arangodb --local` diff --git a/test/process_cluster_recovery_test.go b/test/process_cluster_recovery_test.go index 990f3113..530689c4 100644 --- a/test/process_cluster_recovery_test.go +++ b/test/process_cluster_recovery_test.go @@ -36,6 +36,8 @@ import ( // TestProcessClusterRecovery starts a master starter, followed by 2 slave starters. // Once started, it destroys one of the starters and attempts a recovery. func TestProcessClusterRecovery(t *testing.T) { + SkipOnTravis(t, "Test does not work on TRAVIS VM") // TODO: Fix needed + removeArangodProcesses(t) needTestMode(t, testModeProcess) needStarterMode(t, starterModeCluster) @@ -45,17 +47,17 @@ func TestProcessClusterRecovery(t *testing.T) { start := time.Now() master := Spawn(t, "${STARTER} --starter.port=8528 "+createEnvironmentStarterOptions()) - defer master.Close() + defer closeProcess(t, master, "Master") dataDirSlave1 := SetUniqueDataDir(t) defer os.RemoveAll(dataDirSlave1) slave1 := Spawn(t, "${STARTER} --starter.port=8628 --starter.join 127.0.0.1:8528 "+createEnvironmentStarterOptions()) - defer slave1.Close() + defer closeProcess(t, slave1, "Slave1") dataDirSlave2 := SetUniqueDataDir(t) defer os.RemoveAll(dataDirSlave2) slave2 := Spawn(t, "${STARTER} --starter.port=8728 --starter.join 127.0.0.1:8528 "+createEnvironmentStarterOptions()) - defer slave2.Close() + defer closeProcess(t, slave2, "Slave2") if ok := WaitUntilStarterReady(t, whatCluster, 3, master, slave1, slave2); ok { t.Logf("Cluster start took %s", time.Since(start)) @@ -105,11 +107,11 @@ func TestProcessClusterRecovery(t *testing.T) { // Restart slave1 os.Setenv("DATA_DIR", dataDirSlave1) - slave1 = Spawn(t, "${STARTER} --starter.port=8628 --starter.join 127.0.0.1:8528 "+createEnvironmentStarterOptions()) - defer slave1.Close() + master = Spawn(t, "${STARTER} --starter.port=8628 --starter.join 127.0.0.1:8528 "+createEnvironmentStarterOptions()) + defer closeProcess(t, master, "Master 2") // Wait until recovered - if ok := WaitUntilStarterReady(t, whatCluster, 3, slave1); ok { + if ok := WaitUntilStarterReady(t, whatCluster, 3, master, slave1, slave2); ok { t.Logf("Cluster start (with recovery) took %s", time.Since(start)) testCluster(t, insecureStarterEndpoint(0), false) testCluster(t, insecureStarterEndpoint(100), false) diff --git a/test/process_cluster_resign_leadership_test.go b/test/process_cluster_resign_leadership_test.go index 8803e4de..0fb39cb8 100644 --- a/test/process_cluster_resign_leadership_test.go +++ b/test/process_cluster_resign_leadership_test.go @@ -38,6 +38,8 @@ import ( // TestProcessClusterResignLeadership starts a master starter, followed by 2 slave starters. // It closes the starter where the leader of the shard resides and check whether new leader of the shard is elected. func TestProcessClusterResignLeadership(t *testing.T) { + log := GetLogger(t) + removeArangodProcesses(t) needTestMode(t, testModeProcess) needStarterMode(t, starterModeCluster) @@ -80,6 +82,12 @@ func TestProcessClusterResignLeadership(t *testing.T) { databaseName := "_system" collectionName := "test" + + WaitUntilServiceReadyAPI(t, coordinatorClient, func(t *testing.T, ctx context.Context, c driver.Client) error { + _, err := coordinatorClient.Database(context.Background(), databaseName) + return err + }).ExecuteT(t, 15*time.Second, 500*time.Millisecond) + database, err := coordinatorClient.Database(context.Background(), databaseName) if err != nil { t.Fatal(err.Error()) @@ -163,29 +171,39 @@ func TestProcessClusterResignLeadership(t *testing.T) { } }() - ShutdownStarter(t, starterEndpointWithLeader) + waitForCallFunction(t, ShutdownStarterCall(starterEndpointWithLeader)) + cancel() wg.Wait() if errRead != nil { t.Logf("Reading documents: %s", errRead.Error()) } - // check new leader of the shard. - newDBServerLeader, err := getServerIDLeaderForFirstShard(coordinatorClient, database, collectionName) - if err != nil { - t.Fatal(err.Error()) - } + log.Log("Waiting for shutdown of services") - if dbServerLeader == newDBServerLeader { - t.Fatalf("DB server's ID '%s' can not be the same after leadership resignation", dbServerLeader) - } + NewTimeoutFunc(func() error { + // check new leader of the shard. + newDBServerLeader, err := getServerIDLeaderForFirstShard(coordinatorClient, database, collectionName) + if err != nil { + log.Log("Error while fetching shard details: %s", err.Error()) + return nil + } + + if dbServerLeader == newDBServerLeader { + log.Log("Shard leader is on same server %s", dbServerLeader) + return nil + } + + return NewInterrupt() + }).ExecuteT(t, time.Minute, 500*time.Millisecond) // close the rest of the starters. for _, endpoint := range starterEndpoints { if endpoint == starterEndpointWithLeader { continue } - ShutdownStarter(t, endpoint) + waitForCallFunction(t, + ShutdownStarterCall(endpoint)) } } diff --git a/test/process_cluster_upgrade_test.go b/test/process_cluster_upgrade_test.go index 21326f6f..99f39374 100644 --- a/test/process_cluster_upgrade_test.go +++ b/test/process_cluster_upgrade_test.go @@ -27,6 +27,9 @@ import ( "os" "testing" "time" + + "github.com/arangodb-helper/arangodb/client" + "github.com/arangodb/go-driver" ) // TestProcessClusterUpgrade starts a master starter, followed by 2 slave starters. @@ -72,6 +75,21 @@ func testUpgradeProcess(t *testing.T, endpoint string) { t.Log("Starting database upgrade") c := NewStarterClient(t, endpoint) ctx := context.Background() + + waitForStarter(t, c) + + auth := driver.BasicAuthentication("root", "") + starterEndpointForCoordinator := insecureStarterEndpoint(1 * portIncrement) + coordinatorClient, err := CreateClient(t, starterEndpointForCoordinator, client.ServerTypeCoordinator, auth) + if err != nil { + t.Fatal(err.Error()) + } + + WaitUntilServiceReadyAPI(t, coordinatorClient, func(t *testing.T, ctx context.Context, c driver.Client) error { + _, err := coordinatorClient.Database(context.Background(), "_system") + return err + }).ExecuteT(t, 15*time.Second, 500*time.Millisecond) + if err := c.StartDatabaseUpgrade(ctx, false); err != nil { t.Fatalf("StartDatabaseUpgrade failed: %v", err) } diff --git a/test/process_resilientsingle_default_test.go b/test/process_resilientsingle_default_test.go index 6d5997ef..eed914a3 100644 --- a/test/process_resilientsingle_default_test.go +++ b/test/process_resilientsingle_default_test.go @@ -96,10 +96,8 @@ func TestProcessResilientSingleDefaultShutdownViaAPI(t *testing.T) { testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement)) - ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)), + ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement))) } diff --git a/test/process_single_test.go b/test/process_single_test.go index 3f9bd597..4c8e8754 100644 --- a/test/process_single_test.go +++ b/test/process_single_test.go @@ -70,10 +70,8 @@ func TestProcessSingleShutdownViaAPI(t *testing.T) { testSingle(t, insecureStarterEndpoint(0*portIncrement), false) } - if isVerbose { - t.Log("Waiting for termination") - } - ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement)) + waitForCallFunction(t, + ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement))) } // TestProcessSingleAutoKeyFile runs `arangodb --starter.mode=single --ssl.auto-key` diff --git a/test/process_util.go b/test/process_util.go index 8f5ac310..5903d15e 100644 --- a/test/process_util.go +++ b/test/process_util.go @@ -31,3 +31,27 @@ func removeArangodProcesses(t *testing.T) { defer c.Close() c.Wait() } + +func closeProcess(t *testing.T, s *SubProcess, name string) { + s.Close() + + showProcessLogs(t, s, name) +} + +func listArangodProcesses(t *testing.T, log Logger) { + c := SpawnWithExpand(t, "pidof -x arangod", false) + defer c.Close() + c.Wait() + + logProcessOutput(log, c, "Processes: ") +} + +func showProcessLogs(t *testing.T, s *SubProcess, name string) { + if !t.Failed() { + return + } + + log := GetLogger(t) + + logProcessOutput(log, s, "Log of process: %s", name) +} diff --git a/test/server_util.go b/test/server_util.go index 47f8e9eb..b3b0c495 100644 --- a/test/server_util.go +++ b/test/server_util.go @@ -106,13 +106,35 @@ func testResilientSingle(t *testing.T, starterEndpoint string, isSecure bool, ex return c } +// waitForStarter waits when starter endpoint starts responding +func waitForStarter(t *testing.T, c client.API) { + throttle := NewThrottle(2 * time.Second) + NewTimeoutFunc(func() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := c.Version(ctx); err != nil { + throttle.Execute(func() { + t.Logf("Version check failed due to %s", err.Error()) + }) + return nil + } else { + return NewInterrupt() + } + }).ExecuteT(t, time.Minute, 500*time.Millisecond) +} + // testProcesses runs a series of tests to verify a good series of database servers. func testProcesses(t *testing.T, c client.API, mode, starterEndpoint string, isSecure bool, expectAgencyOnly bool, syncEnabled bool, singleTimeout, reachableTimeout time.Duration) { // Give the deployment a little bit of time: - time.Sleep(3 * time.Second) ctx := context.Background() + // Wait until starter restarts + waitForStarter(t, c) + log := GetLogger(t) + + log.Log("Starter is responding: %s", starterEndpoint) + // Fetch version if info, err := c.Version(ctx); err != nil { t.Errorf("Failed to get starter version: %s", describe(err)) diff --git a/test/timeout.go b/test/timeout.go new file mode 100644 index 00000000..b76ea43a --- /dev/null +++ b/test/timeout.go @@ -0,0 +1,130 @@ +// +// DISCLAIMER +// +// Copyright 2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package test + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func NewThrottle(interval time.Duration) Throttle { + return &throttle{ + interval: interval, + } +} + +type Throttle interface { + Execute(func()) +} + +type throttle struct { + last time.Time + interval time.Duration +} + +func (t *throttle) Execute(f func()) { + n := time.Now() + if n.After(t.last.Add(t.interval)) { + f() + t.last = n + } +} + +type TimeoutFunc func() error + +func NewTimeoutFunc(f func() error) TimeoutFunc { + return f +} + +func (f TimeoutFunc) Execute(timeout, interval time.Duration) error { + if err := f(); err != nil { + if IsInterrupt(err) { + return nil + } + + return err + } + + timeoutT := time.NewTimer(timeout) + defer timeoutT.Stop() + + intervalT := time.NewTicker(interval) + defer intervalT.Stop() + + for { + select { + case <-timeoutT.C: + return fmt.Errorf("timeout") + case <-intervalT.C: + if err := f(); err != nil { + if IsInterrupt(err) { + return nil + } + + return err + } + } + } +} + +func (f TimeoutFunc) ExecuteWithLog(log Logger, timeout, interval time.Duration) (err error) { + now := time.Now() + + defer func() { + if err == nil { + log.Log("Success - took %s", time.Now().Sub(now).String()) + } else { + log.Log("Error - took %s - %s", time.Now().Sub(now).String(), err.Error()) + } + }() + + err = f.Execute(timeout, interval) + return +} + +func (f TimeoutFunc) ExecuteTWithLog(t *testing.T, log Logger, timeout, interval time.Duration) { + require.NoError(t, f.ExecuteWithLog(log, timeout, interval)) +} + +func (f TimeoutFunc) ExecuteT(t *testing.T, timeout, interval time.Duration) { + require.NoError(t, f.Execute(timeout, interval)) +} + +type Interrupt struct { +} + +func (i Interrupt) Error() string { + return "interrupt" +} + +func NewInterrupt() error { + return Interrupt{} +} + +func IsInterrupt(err error) bool { + _, ok := err.(Interrupt) + return ok +} diff --git a/test/util.go b/test/util.go index 5bf96eb2..a6a03e7e 100644 --- a/test/util.go +++ b/test/util.go @@ -23,10 +23,13 @@ package test import ( + "bufio" + "bytes" "context" "encoding/json" "fmt" "io/ioutil" + "net/http" "net/url" "os" "regexp" @@ -35,24 +38,43 @@ import ( "testing" "time" + "github.com/arangodb/go-driver" + "github.com/arangodb-helper/arangodb/client" shell "github.com/kballard/go-shellquote" "github.com/pkg/errors" ) const ( - ctrlC = "\u0003" - whatCluster = "cluster" - whatSingle = "single server" - whatResilientSingle = "resilient single server" - testModeProcess = "localprocess" - testModeDocker = "docker" - starterModeCluster = "cluster" - starterModeSingle = "single" - starterModeActiveFailover = "activefailover" - portIncrement = 10 + ctrlC = "\u0003" + whatCluster = "cluster" + whatSingle = "single server" + whatResilientSingle = "resilient single server" + testModeProcess = "localprocess" + testModeDocker = "docker" + starterModeCluster = "cluster" + starterModeSingle = "single" + starterModeActiveFailover = "activefailover" + portIncrement = 10 + travisEnv EnvironmentVariable = "TRAVIS" ) +type EnvironmentVariable string + +func (e EnvironmentVariable) String() string { + return string(e) +} + +func (e EnvironmentVariable) Lookup() (string, bool) { + return os.LookupEnv(e.String()) +} + +func SkipOnTravis(t *testing.T, format string, args ...interface{}) { + if _, ok := travisEnv.Lookup(); ok { + t.Skipf(format, args...) + } +} + var ( isVerbose bool isEnterprise bool @@ -144,53 +166,41 @@ func SetUniqueDataDir(t *testing.T) string { return dataDir } -type waitUntilReadyResult struct { - Ready bool - TimeSpan time.Duration - Message string -} - // WaitUntilStarterReady waits until all given starter processes have reached the "Your cluster is ready state" func WaitUntilStarterReady(t *testing.T, what string, requiredGoodResults int, starters ...*SubProcess) bool { - results := make(chan waitUntilReadyResult, len(starters)) + results := make([]error, len(starters)) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() - for index, starter := range starters { - starter := starter // Used in nested function - id := fmt.Sprintf("starter-%d", index+1) - go func() { - started := time.Now() - if err := starter.ExpectTimeout(ctx, time.Minute*3, regexp.MustCompile(fmt.Sprintf("Your %s can now be accessed with a browser at", what)), id); err != nil { - timeSpan := time.Since(started) - results <- waitUntilReadyResult{ - Ready: false, - TimeSpan: timeSpan, - Message: fmt.Sprintf("Starter is not ready in time (after %s): %s", timeSpan, describe(err)), - } - } else { - results <- waitUntilReadyResult{ - Ready: true, - } - } - }() + + var wg sync.WaitGroup + + wg.Add(len(results)) + + for id, starter := range starters { + go func(i int, s *SubProcess) { + defer wg.Done() + defer cancel() + id := fmt.Sprintf("starter-%d", i+1) + + results[i] = s.ExpectTimeout(ctx, time.Minute*3, regexp.MustCompile(fmt.Sprintf("Your %s can now be accessed with a browser at", what)), id) + }(id, starter) } - okCount := 0 - errorCount := 0 - errorMessages := make([]string, 0, len(starters)) - for result := range results { - if result.Ready { - okCount++ - } else { - errorCount++ - errorMessages = append(errorMessages, result.Message) - } - if okCount >= requiredGoodResults { - return true - } - if okCount+errorCount == len(starters) { - break + + wg.Wait() + + failed := 0 + for _, result := range results { + if result != nil { + failed++ } } + + if failed <= requiredGoodResults { + GetLogger(t).Log("Starter Started") + return true + } + if os.Getenv("DEBUG_CLUSTER") == "interactive" { // Halt forever fmt.Println("Cluster not ready in time, halting forever for debugging") @@ -198,12 +208,84 @@ func WaitUntilStarterReady(t *testing.T, what string, requiredGoodResults int, s time.Sleep(time.Hour) } } - for _, msg := range errorMessages { + for _, msg := range results { t.Error(msg) } + return false } +type ServiceReadyCheckFunc func(t *testing.T, ctx context.Context, c driver.Client) error +type ServiceReadyCheck func(t *testing.T, c driver.Client, check ServiceReadyCheckFunc) bool + +// WaitUntilServiceReadyRetryOnError do not allow any errors to occur +func WaitUntilServiceReadyRetryOnError(t *testing.T, c driver.Client, check ServiceReadyCheckFunc) bool { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := check(t, ctx, c) + + return err == nil +} + +// WaitUntilServiceReadyRetryOn503 retry on 503 code from service +func WaitUntilServiceReadyRetryOn503(t *testing.T, c driver.Client, check ServiceReadyCheckFunc) bool { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := check(t, ctx, c) + if err == nil { + return true + } + + if ae, ok := driver.AsArangoError(err); !ok { + // Ignore unknown errors + return true + } else { + // Check if 503 is returned + return ae.Code != http.StatusServiceUnavailable + } +} + +// WaitUntilServiceReadyAPI return timeout function which waits until service is fully ready +func WaitUntilServiceReadyAPI(t *testing.T, c driver.Client, check ServiceReadyCheckFunc) TimeoutFunc { + return WaitUntilServiceReady(t, c, check, WaitUntilServiceReadyRetryOn503, WaitUntilServiceReadyRetryOnError) +} + +// WaitUntilServiceReady retry on errors from service +func WaitUntilServiceReady(t *testing.T, c driver.Client, checkFunc ServiceReadyCheckFunc, checks ...ServiceReadyCheck) TimeoutFunc { + return func() error { + for _, check := range checks { + if !check(t, c, checkFunc) { + return nil + } + } + + return Interrupt{} + } +} + +func WaitForHttpPortClosed(log Logger, throttle Throttle, url string) TimeoutFunc { + return func() error { + _, err := http.Get(url) + if err == nil { + throttle.Execute(func() { + log.Log("Got empty response") + }) + return nil + } + + if strings.Contains(err.Error(), "connection refused") { + return NewInterrupt() + } + + throttle.Execute(func() { + log.Log("Unknown error: %s", err.Error()) + }) + return nil + } +} + // SendIntrAndWait stops all all given starter processes by sending a Ctrl-C into it. // It then waits until the process has terminated. func SendIntrAndWait(t *testing.T, starters ...*SubProcess) bool { @@ -257,11 +339,26 @@ func NewStarterClient(t *testing.T, endpoint string) client.API { return c } -// ShutdownStarter calls the starter the shutdown via the HTTP API. -func ShutdownStarter(t *testing.T, endpoint string) { +// ShutdownStarterCall returns function representation of ShutdownStarter. +func ShutdownStarterCall(endpoint string) callFunction { + return func(t *testing.T) { + shutdownStarter(t, endpoint) + } +} + +// shutdownStarter calls the starter the shutdown via the HTTP API. +func shutdownStarter(t *testing.T, endpoint string) { + log := GetLogger(t) + + log.Log("Terminating %s", endpoint) + + defer func() { + log.Log("Terminated %s", endpoint) + }() + c := NewStarterClient(t, endpoint) if err := c.Shutdown(context.Background(), false); err != nil { - t.Errorf("Shutdown failed: %s", describe(err)) + log.Log("Shutdown failed: %s", describe(err)) } WaitUntilStarterGone(t, endpoint) } @@ -301,3 +398,43 @@ func createLicenseKeyOption() string { } return "" } + +type callFunction func(t *testing.T) + +func waitForCallFunction(t *testing.T, funcs ...callFunction) { + var wg sync.WaitGroup + + wg.Add(len(funcs)) + + for _, f := range funcs { + go func(z callFunction) { + defer wg.Done() + z(t) + }(f) + } + + wg.Wait() +} + +func logProcessOutput(log Logger, p *SubProcess, prefix string, args ...interface{}) { + pre := "" + if prefix != "" { + pre = fmt.Sprintf(prefix, args...) + } + + reader := bufio.NewReader(bytes.NewReader(p.Output())) + + for { + line, _, err := reader.ReadLine() + if len(line) > 0 { + if pre != "" { + log.Log(string(line)) + } else { + log.Log("%s%s", pre, string(line)) + } + } + if err != nil { + break + } + } +}