From 25a5c493b0e66c2e0f68cfbeb646f2301a30d959 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Mon, 7 Jun 2021 14:58:33 +0000
Subject: [PATCH 01/37] Use GCR in tests

---
 .travis.yml | 12 ++++++------
 Dockerfile  |  3 ++-
 Makefile    |  4 +++-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 8e951f37..cf1fd8ef 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,12 +9,12 @@ go:
   - "1.13.x"
 
 env:
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process ARANGODB=arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        ARANGODB=arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process ARANGODB=arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        ARANGODB=arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process ARANGODB=arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        ARANGODB=arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
 
 script: make $TEST_SUITE
 
diff --git a/Dockerfile b/Dockerfile
index 650cfcc2..026d056f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,5 @@
-FROM alpine:3.11
+ARG IMAGE=alpine:3.11
+FROM ${IMAGE}
 MAINTAINER Max Neunhoeffer <max@arangodb.com>
 
 COPY bin/linux/amd64/arangodb /app/
diff --git a/Makefile b/Makefile
index 8f936b58..44277cb6 100644
--- a/Makefile
+++ b/Makefile
@@ -10,6 +10,8 @@ VERSION_MAJOR := $(shell echo $(VERSION_MAJOR_MINOR) | cut -f 1 -d '.')
 COMMIT := $(shell git rev-parse --short HEAD)
 MAKEFILE := $(ROOTDIR)/Makefile
 
+ALPINE_IMAGE ?= alpine:3.11
+
 DOCKERCLI ?= $(shell which docker)
 GOBUILDLINKTARGET := ../../../..
 
@@ -149,7 +151,7 @@ $(TESTBIN): $(GOBUILDDIR) $(TEST_SOURCES) $(BIN)
 	$(DOCKER_CMD) go test -c -o "$(TEST_BIN)" ./test
 
 docker: build
-	$(DOCKERCLI) build -t arangodb/arangodb-starter .
+	$(DOCKERCLI) build -t arangodb/arangodb-starter --build-arg "IMAGE=$(ALPINE_IMAGE)" .
 
 docker-push: docker
 ifneq ($(DOCKERNAMESPACE), arangodb)

From 42a41e91dbf777f18b94f44a942138663b6d8207 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Tue, 8 Jun 2021 10:27:50 +0000
Subject: [PATCH 02/37] Fix sleep

---
 go.mod              |  1 +
 test/server_util.go | 17 ++++++++-
 test/timeout.go     | 88 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100644 test/timeout.go

diff --git a/go.mod b/go.mod
index 54a1f3c9..92fe614d 100644
--- a/go.mod
+++ b/go.mod
@@ -29,6 +29,7 @@ require (
 	github.com/ryanuber/columnize v2.1.0+incompatible
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.5.1
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
 	github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 // indirect
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
diff --git a/test/server_util.go b/test/server_util.go
index 47f8e9eb..17f9a0d7 100644
--- a/test/server_util.go
+++ b/test/server_util.go
@@ -106,13 +106,28 @@ func testResilientSingle(t *testing.T, starterEndpoint string, isSecure bool, ex
 	return c
 }
 
+// waitForStarter waits when starter endpoint starts responding
+func waitForStarter(t *testing.T, c client.API) {
+	NewTimeoutFunc(func() error {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		if _, err := c.Version(ctx); err != nil {
+			return nil
+		} else {
+			return NewInterrupt()
+		}
+	}).ExecuteT(t, time.Minute, 500*time.Millisecond)
+}
+
 // testProcesses runs a series of tests to verify a good series of database servers.
 func testProcesses(t *testing.T, c client.API, mode, starterEndpoint string, isSecure bool,
 	expectAgencyOnly bool, syncEnabled bool, singleTimeout, reachableTimeout time.Duration) {
 	// Give the deployment a little bit of time:
-	time.Sleep(3 * time.Second)
 	ctx := context.Background()
 
+	// Wait until starter restarts
+	waitForStarter(t, c)
+
 	// Fetch version
 	if info, err := c.Version(ctx); err != nil {
 		t.Errorf("Failed to get starter version: %s", describe(err))
diff --git a/test/timeout.go b/test/timeout.go
new file mode 100644
index 00000000..bb5b8d9a
--- /dev/null
+++ b/test/timeout.go
@@ -0,0 +1,88 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+type TimeoutFunc func() error
+
+func NewTimeoutFunc(f func() error) TimeoutFunc {
+	return f
+}
+
+func (f TimeoutFunc) Execute(timeout, interval time.Duration) error {
+	if err := f(); err != nil {
+		if IsInterrupt(err) {
+			return nil
+		}
+
+		return err
+	}
+
+	timeoutT := time.NewTimer(timeout)
+	defer timeoutT.Stop()
+
+	intervalT := time.NewTicker(interval)
+	defer intervalT.Stop()
+
+	for {
+		select {
+		case <-timeoutT.C:
+			return fmt.Errorf("timeout")
+		case <-intervalT.C:
+			if err := f(); err != nil {
+				if IsInterrupt(err) {
+					return nil
+				}
+
+				return err
+			}
+		}
+	}
+}
+
+func (f TimeoutFunc) ExecuteT(t *testing.T, timeout, interval time.Duration) {
+	require.NoError(t, f.Execute(timeout, interval))
+}
+
+type Interrupt struct {
+}
+
+func (i Interrupt) Error() string {
+	return "interrupt"
+}
+
+func NewInterrupt() error {
+	return Interrupt{}
+}
+
+func IsInterrupt(err error) bool {
+	_, ok := err.(Interrupt)
+	return ok
+}

From 63077d86d14ca155ed60fda5c362dcd1a42daa9b Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Tue, 8 Jun 2021 14:56:51 +0000
Subject: [PATCH 03/37] Add more logs

---
 test/server_util.go |  4 ++++
 test/timeout.go     | 23 +++++++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/test/server_util.go b/test/server_util.go
index 17f9a0d7..4b5a6936 100644
--- a/test/server_util.go
+++ b/test/server_util.go
@@ -108,10 +108,14 @@ func testResilientSingle(t *testing.T, starterEndpoint string, isSecure bool, ex
 
 // waitForStarter waits when starter endpoint starts responding
 func waitForStarter(t *testing.T, c client.API) {
+	throttle := NewThrottle(2 * time.Second)
 	NewTimeoutFunc(func() error {
 		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 		defer cancel()
 		if _, err := c.Version(ctx); err != nil {
+			throttle.Execute(func() {
+				t.Logf("Version check failed due to %s", err.Error())
+			})
 			return nil
 		} else {
 			return NewInterrupt()
diff --git a/test/timeout.go b/test/timeout.go
index bb5b8d9a..987a1d01 100644
--- a/test/timeout.go
+++ b/test/timeout.go
@@ -30,6 +30,29 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func NewThrottle(interval time.Duration) Throttle {
+	return &throttle{
+		interval: interval,
+	}
+}
+
+type Throttle interface {
+	Execute(func())
+}
+
+type throttle struct {
+	last     time.Time
+	interval time.Duration
+}
+
+func (t *throttle) Execute(f func()) {
+	n := time.Now()
+	if n.After(t.last.Add(t.interval)) {
+		f()
+		t.last = n
+	}
+}
+
 type TimeoutFunc func() error
 
 func NewTimeoutFunc(f func() error) TimeoutFunc {

From d49712afff0a7036e5cea1623f9271198384322a Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Tue, 8 Jun 2021 17:30:04 +0000
Subject: [PATCH 04/37] Add more logs

---
 .travis.yml | 12 ++++++------
 Makefile    |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index cf1fd8ef..01f53ef3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,12 +9,12 @@ go:
   - "1.13.x"
 
 env:
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
 
 script: make $TEST_SUITE
 
diff --git a/Makefile b/Makefile
index 44277cb6..cf6348bc 100644
--- a/Makefile
+++ b/Makefile
@@ -30,7 +30,7 @@ REPOPATH := $(ORGPATH)/$(REPONAME)
 
 GOPATH := $(GOBUILDDIR)
 GOVERSION := 1.13.6
-GOIMAGE := golang:$(GOVERSION)
+GOIMAGE ?= golang:$(GOVERSION)
 
 GOOS ?= linux
 GOARCH ?= amd64

From 240bd0b5feffa9b73b2645eb282233b71e4cc3a0 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 08:30:55 +0000
Subject: [PATCH 05/37] Reduce tests

---
 .travis.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 01f53ef3..97d54b14 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,11 +10,11 @@ go:
 
 env:
   - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
+#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
   - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
+#  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
 
 script: make $TEST_SUITE
 

From 135ed6e3482b55b059928dbec23e01fef00a9f97 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 11:43:38 +0000
Subject: [PATCH 06/37] change process termination

---
 .travis.yml                                   |   8 +-
 pkg/definitions/server_type.go                |  11 ++
 service/process_wrapper.go                    |   2 +-
 test/docker_activefailover_default_test.go    |  23 ++--
 test/docker_activefailover_local_test.go      |  12 +-
 test/docker_cluster_default_test.go           |  17 ++-
 test/docker_cluster_diff_logdir_test.go       |  20 ++--
 test/docker_cluster_diff_ports_test.go        |  10 +-
 test/docker_cluster_local_test.go             |  18 +--
 test/docker_cluster_multi_join_test.go        |  10 +-
 test/docker_cluster_recovery_test.go          |  10 +-
 test/docker_cluster_sync_test.go              |  10 +-
 test/docker_cluster_upgrade_test.go           |  10 +-
 test/docker_database_version_test.go          |   3 +-
 test/docker_resilientsingle_default_test.go   |  20 ++--
 test/docker_resilientsingle_local_test.go     |  12 +-
 test/docker_single_test.go                    |  30 ++---
 test/gexpect.go                               |  47 +++++---
 test/log.go                                   | 111 ++++++++++++++++++
 test/process_activefailover_default_test.go   |  10 +-
 test/process_cluster_default_test.go          |  10 +-
 test/process_cluster_local_test.go            |   6 +-
 .../process_cluster_resign_leadership_test.go |   7 +-
 test/process_resilientsingle_default_test.go  |  10 +-
 test/process_single_test.go                   |   6 +-
 test/util.go                                  | 109 ++++++++++-------
 26 files changed, 322 insertions(+), 220 deletions(-)
 create mode 100644 test/log.go

diff --git a/.travis.yml b/.travis.yml
index 97d54b14..01f53ef3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,11 +10,11 @@ go:
 
 env:
   - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
-#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
   - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-#  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
-#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
 
 script: make $TEST_SUITE
 
diff --git a/pkg/definitions/server_type.go b/pkg/definitions/server_type.go
index 679c914d..d63858f4 100644
--- a/pkg/definitions/server_type.go
+++ b/pkg/definitions/server_type.go
@@ -24,6 +24,7 @@ package definitions
 
 import (
 	"fmt"
+	"time"
 )
 
 // ServerType specifies the types of database servers.
@@ -63,6 +64,16 @@ func (s ServerType) PortOffset() int {
 	}
 }
 
+// InitialStopTimeout returns initial delay for process stopping
+func (s ServerType) InitialStopTimeout() time.Duration {
+	switch s {
+	case ServerTypeDBServer, ServerTypeSingle, ServerTypeResilientSingle:
+		return 3 * time.Second
+	default:
+		return time.Second
+	}
+}
+
 // ProcessType returns the type of process needed to run a server of given type.
 func (s ServerType) ProcessType() ProcessType {
 	switch s {
diff --git a/service/process_wrapper.go b/service/process_wrapper.go
index fbcc8918..86219527 100644
--- a/service/process_wrapper.go
+++ b/service/process_wrapper.go
@@ -229,7 +229,7 @@ func (p *processWrapper) run(startedCh chan<- struct{}) {
 			case <-p.stopping:
 				if p.s.stopping {
 					// Starter is being closed
-					terminateProcessWithActions(logProcess, p.proc, p.serverType, 10*time.Second, time.Minute)
+					terminateProcessWithActions(logProcess, p.proc, p.serverType, time.Second, time.Minute)
 				} else {
 					// Process restart
 					terminateProcessWithActions(logProcess, p.proc, p.serverType, 0, time.Minute)
diff --git a/test/docker_activefailover_default_test.go b/test/docker_activefailover_default_test.go
index 7c6cb056..2174fe0e 100644
--- a/test/docker_activefailover_default_test.go
+++ b/test/docker_activefailover_default_test.go
@@ -33,6 +33,9 @@ import (
 // TestDockerActiveFailoverDefault runs 3 arangodb starters in docker with mode=activefailover
 // and otherwise default settings.
 func TestDockerActiveFailoverDefault(t *testing.T) {
+	log := GetLogger(t)
+	defer log.Clean()
+
 	needTestMode(t, testModeDocker)
 	needStarterMode(t, starterModeActiveFailover)
 	if os.Getenv("IP") == "" {
@@ -48,6 +51,7 @@ func TestDockerActiveFailoverDefault(t *testing.T) {
 			--starter.address=$IP \
 			--starter.mode=activefailover
 	*/
+
 	volID1 := createDockerID("vol-starter-test-activefailover-default1-")
 	createDockerVolume(t, volID1)
 	defer removeDockerVolume(t, volID1)
@@ -132,12 +136,9 @@ func TestDockerActiveFailoverDefault(t *testing.T) {
 		testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t, ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 }
 
 // TestDockerActiveFailover2Instance runs 3 arangodb starters in docker with mode=activefailover
@@ -243,10 +244,8 @@ func TestDockerActiveFailover2Instance(t *testing.T) {
 		testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, true)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 }
diff --git a/test/docker_activefailover_local_test.go b/test/docker_activefailover_local_test.go
index 2b0607e7..cdd17284 100644
--- a/test/docker_activefailover_local_test.go
+++ b/test/docker_activefailover_local_test.go
@@ -83,10 +83,8 @@ func TestDockerActiveFailoverLocal(t *testing.T) {
 		testResilientSingle(t, insecureStarterEndpoint(0*portIncrement), false, false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)))
 }
 
 // TestDockerActiveFailoverSecure runs the arangodb starter in docker with mode `activefailover`,
@@ -144,8 +142,6 @@ func TestDockerResilientActiveFailoverSecure(t *testing.T) {
 		testResilientSingle(t, secureStarterEndpoint(0*portIncrement), true, false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, secureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(secureStarterEndpoint(0*portIncrement)))
 }
diff --git a/test/docker_cluster_default_test.go b/test/docker_cluster_default_test.go
index 0fbcf9e8..c980c9bd 100644
--- a/test/docker_cluster_default_test.go
+++ b/test/docker_cluster_default_test.go
@@ -127,12 +127,10 @@ func TestDockerClusterDefault(t *testing.T) {
 		testCluster(t, insecureStarterEndpoint(2*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 }
 
 // TestOldDockerClusterDefault runs 3 arangodb starters in docker with default settings.
@@ -235,7 +233,8 @@ func TestOldDockerClusterDefault(t *testing.T) {
 	if isVerbose {
 		t.Log("Waiting for termination")
 	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 }
diff --git a/test/docker_cluster_diff_logdir_test.go b/test/docker_cluster_diff_logdir_test.go
index 9af9744e..265bb6f8 100644
--- a/test/docker_cluster_diff_logdir_test.go
+++ b/test/docker_cluster_diff_logdir_test.go
@@ -145,12 +145,10 @@ func TestDockerClusterDifferentLogDir(t *testing.T) {
 		testCluster(t, insecureStarterEndpoint(2*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 
 	check := func(rootDir string, expectedFileCount int) {
 		files, err := getRecursiveLogFiles(rootDir)
@@ -282,12 +280,10 @@ func TestDockerClusterDifferentLogDirNoLog2File(t *testing.T) {
 		testCluster(t, insecureStarterEndpoint(2*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 
 	check := func(rootDir string, expectedFileCount int) {
 		files, err := getRecursiveLogFiles(rootDir)
diff --git a/test/docker_cluster_diff_ports_test.go b/test/docker_cluster_diff_ports_test.go
index 4723cc68..7d10c916 100644
--- a/test/docker_cluster_diff_ports_test.go
+++ b/test/docker_cluster_diff_ports_test.go
@@ -131,10 +131,8 @@ func TestDockerClusterDifferentPorts(t *testing.T) {
 		testCluster(t, "http://localhost:8000", false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, "http://localhost:6000")
-	ShutdownStarter(t, "http://localhost:7000")
-	ShutdownStarter(t, "http://localhost:8000")
+	waitForCallFunction(t,
+		ShutdownStarterCall("http://localhost:6000"),
+		ShutdownStarterCall("http://localhost:7000"),
+		ShutdownStarterCall("http://localhost:8000"))
 }
diff --git a/test/docker_cluster_local_test.go b/test/docker_cluster_local_test.go
index 3f2ff1d4..b798a0a6 100644
--- a/test/docker_cluster_local_test.go
+++ b/test/docker_cluster_local_test.go
@@ -81,10 +81,8 @@ func TestDockerClusterLocal(t *testing.T) {
 		testCluster(t, insecureStarterEndpoint(0*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)))
 }
 
 // TestDockerClusterLocalAgencySize1 runs the arangodb starter in docker
@@ -141,10 +139,8 @@ func TestDockerClusterLocalAgencySize1(t *testing.T) {
 		testCluster(t, insecureStarterEndpoint(0*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)))
 }
 
 // TestOldDockerClusterLocal runs the arangodb starter in docker with `--local`
@@ -197,8 +193,6 @@ func TestOldDockerClusterLocal(t *testing.T) {
 		testCluster(t, insecureStarterEndpoint(0*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)))
 }
diff --git a/test/docker_cluster_multi_join_test.go b/test/docker_cluster_multi_join_test.go
index ea62468e..57f6c947 100644
--- a/test/docker_cluster_multi_join_test.go
+++ b/test/docker_cluster_multi_join_test.go
@@ -134,10 +134,8 @@ func TestDockerClusterMultipleJoins(t *testing.T) {
 		testCluster(t, "http://localhost:8000", false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, "http://localhost:6000")
-	ShutdownStarter(t, "http://localhost:7000")
-	ShutdownStarter(t, "http://localhost:8000")
+	waitForCallFunction(t,
+		ShutdownStarterCall("http://localhost:6000"),
+		ShutdownStarterCall("http://localhost:7000"),
+		ShutdownStarterCall("http://localhost:8000"))
 }
diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index 35e8c8be..4a54cd5e 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -212,10 +212,8 @@ func TestDockerClusterRecovery(t *testing.T) {
 		t.Errorf("Expected RECOVERY file to not-exist, got: %s", describe(err))
 	}*/
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0))
-	ShutdownStarter(t, insecureStarterEndpoint(100))
-	ShutdownStarter(t, insecureStarterEndpoint(200))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0)),
+		ShutdownStarterCall(insecureStarterEndpoint(100)),
+		ShutdownStarterCall(insecureStarterEndpoint(200)))
 }
diff --git a/test/docker_cluster_sync_test.go b/test/docker_cluster_sync_test.go
index 51428c81..659b01db 100644
--- a/test/docker_cluster_sync_test.go
+++ b/test/docker_cluster_sync_test.go
@@ -149,10 +149,8 @@ func TestDockerClusterSync(t *testing.T) {
 		testClusterWithSync(t, insecureStarterEndpoint(2*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 }
diff --git a/test/docker_cluster_upgrade_test.go b/test/docker_cluster_upgrade_test.go
index 7aafead9..d0c0e5e5 100644
--- a/test/docker_cluster_upgrade_test.go
+++ b/test/docker_cluster_upgrade_test.go
@@ -130,10 +130,8 @@ func TestDockerClusterUpgrade(t *testing.T) {
 
 	testUpgradeProcess(t, insecureStarterEndpoint(0*portIncrement))
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 }
diff --git a/test/docker_database_version_test.go b/test/docker_database_version_test.go
index 5648c11c..16da5710 100644
--- a/test/docker_database_version_test.go
+++ b/test/docker_database_version_test.go
@@ -95,5 +95,6 @@ func TestDockerDatabaseVersion(t *testing.T) {
 		t.Logf("Got database-version %s", v)
 	}
 
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)))
 }
diff --git a/test/docker_resilientsingle_default_test.go b/test/docker_resilientsingle_default_test.go
index b347ebf0..d9ab15b6 100644
--- a/test/docker_resilientsingle_default_test.go
+++ b/test/docker_resilientsingle_default_test.go
@@ -132,12 +132,10 @@ func TestDockerResilientSingleDefault(t *testing.T) {
 		testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 }
 
 // TestDockerResilientSingle2Instance runs 3 arangodb starters in docker with mode=resilientsingle
@@ -243,10 +241,8 @@ func TestDockerResilientSingle2Instance(t *testing.T) {
 		testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, true)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 }
diff --git a/test/docker_resilientsingle_local_test.go b/test/docker_resilientsingle_local_test.go
index 351aff95..ca6e95b1 100644
--- a/test/docker_resilientsingle_local_test.go
+++ b/test/docker_resilientsingle_local_test.go
@@ -83,10 +83,8 @@ func TestDockerResilientSingleLocal(t *testing.T) {
 		testResilientSingle(t, insecureStarterEndpoint(0*portIncrement), false, false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)))
 }
 
 // TestDockerResilientSingleLocalSecure runs the arangodb starter in docker with mode `resilientsingle`,
@@ -144,8 +142,6 @@ func TestDockerResilientSingleLocalSecure(t *testing.T) {
 		testResilientSingle(t, secureStarterEndpoint(0*portIncrement), true, false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, secureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(secureStarterEndpoint(0*portIncrement)))
 }
diff --git a/test/docker_single_test.go b/test/docker_single_test.go
index 5e761cf2..36b5319a 100644
--- a/test/docker_single_test.go
+++ b/test/docker_single_test.go
@@ -81,10 +81,8 @@ func TestDockerSingle(t *testing.T) {
 		testSingle(t, insecureStarterEndpoint(0*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)))
 }
 
 // TestDockerSingleAutoKeyFile runs the arangodb starter in docker with `--starter.mode=single` && `--ssl.auto-key`
@@ -140,10 +138,8 @@ func TestDockerSingleAutoKeyFile(t *testing.T) {
 		testSingle(t, secureStarterEndpoint(0*portIncrement), true)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, secureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(secureStarterEndpoint(0*portIncrement)))
 }
 
 // TestDockerSingleAutoContainerName runs the arangodb starter in docker with `--starter.mode=single` automatic detection of its container name.
@@ -195,10 +191,8 @@ func TestDockerSingleAutoContainerName(t *testing.T) {
 		testSingle(t, insecureStarterEndpoint(0*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)))
 }
 
 // TestDockerSingleAutoRocksdb runs the arangodb starter in docker with `--server.storage-engine=rocksdb` and a 3.2+ image.
@@ -253,10 +247,8 @@ func TestDockerSingleAutoRocksdb(t *testing.T) {
 		testSingle(t, insecureStarterEndpoint(0*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)))
 }
 
 // TestOldDockerSingleAutoKeyFile runs the arangodb starter in docker with `--mode=single` && `--sslAutoKeyFile`
@@ -310,8 +302,6 @@ func TestOldDockerSingleAutoKeyFile(t *testing.T) {
 		testSingle(t, secureStarterEndpoint(0*portIncrement), true)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, secureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(secureStarterEndpoint(0*portIncrement)))
 }
diff --git a/test/gexpect.go b/test/gexpect.go
index 15a37b77..22346d61 100644
--- a/test/gexpect.go
+++ b/test/gexpect.go
@@ -85,7 +85,6 @@ func (sp *SubProcess) Start() error {
 		for {
 			n, err := rd.Read(byteBuf)
 			sp.writeOutput(byteBuf[:n])
-			sp.matchExpressions()
 			if err != nil {
 				break
 			}
@@ -158,11 +157,7 @@ func (sp *SubProcess) Wait() error {
 func (sp *SubProcess) ExpectTimeout(ctx context.Context, timeout time.Duration, re *regexp.Regexp, id string) error {
 	found := make(chan struct{})
 
-	sp.mutex.Lock()
-	sp.expressions[re] = found
-	sp.mutex.Unlock()
-
-	sp.matchExpressions()
+	sp.matchExpressionAsync(ctx, found, re)
 
 	select {
 	case <-ctx.Done():
@@ -192,21 +187,35 @@ func (sp *SubProcess) writeOutput(data []byte) {
 	sp.output.Write(data)
 }
 
-func (sp *SubProcess) matchExpressions() {
+func (sp *SubProcess) matchExpressionAsync(ctx context.Context, found chan<- struct{}, regexes ...*regexp.Regexp) {
+	go func() {
+		defer close(found)
+
+		ticker := time.NewTicker(125 * time.Millisecond)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+				if sp.matchExpressionInOutput(regexes...) {
+					return
+				}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+}
+
+func (sp *SubProcess) matchExpressionInOutput(regexes ...*regexp.Regexp) bool {
 	sp.mutex.Lock()
 	defer sp.mutex.Unlock()
-
-	for re, found := range sp.expressions {
-		loc := re.FindIndex(sp.output.Bytes())
-		if loc == nil {
-			// No match
-			continue
+	data := sp.output.Bytes()
+	for _, re := range regexes {
+		if loc := re.FindIndex(data); loc != nil {
+			return true
 		}
-		// Found a match, remove everything until the end of the match
-		n := loc[1]
-		sp.output.Next(n)
-		close(found)
-		// Remove from map
-		delete(sp.expressions, re)
 	}
+
+	return false
 }
diff --git a/test/log.go b/test/log.go
new file mode 100644
index 00000000..cf5f7609
--- /dev/null
+++ b/test/log.go
@@ -0,0 +1,111 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package test
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+)
+
+var (
+	loggerMutex sync.Mutex
+	loggers     = map[*testing.T]Logger{}
+)
+
+func cleanLogger(t *testing.T) {
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+
+	delete(loggers, t)
+}
+
+func getLogger(parent *logger, t *testing.T) Logger {
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+
+	if l, ok := loggers[t]; ok {
+		return l
+	}
+
+	l := &logger{
+		start:  time.Now(),
+		t:      t,
+		parent: parent,
+	}
+
+	loggers[t] = l
+	return l
+}
+
+type Logger interface {
+	Log(format string, args ...interface{})
+
+	SubLogger(t *testing.T) Logger
+
+	Clean()
+}
+
+type logger struct {
+	start time.Time
+	t     *testing.T
+
+	parent *logger
+}
+
+func (l *logger) Clean() {
+	cleanLogger(l.t)
+}
+
+func (l *logger) getParent() *logger {
+	if l == nil || l.parent == nil {
+		return nil
+	}
+
+	if p := l.parent.getParent(); p == nil {
+		return l
+	} else {
+		return p
+	}
+}
+
+func (l *logger) Log(format string, args ...interface{}) {
+	line := fmt.Sprintf(format, args...)
+	if p := l.getParent(); p == nil {
+		line = fmt.Sprintf("Started: %s > %s", time.Now().Sub(l.start), line)
+	} else {
+		line = fmt.Sprintf("Started: %s, In Test: %s > %s", time.Now().Sub(p.start).String(), time.Now().Sub(l.start).String(), line)
+	}
+
+	l.t.Log(line)
+	println(line)
+}
+
+func (l *logger) SubLogger(t *testing.T) Logger {
+	return getLogger(l, t)
+}
+
+func GetLogger(t *testing.T) Logger {
+	return getLogger(nil, t)
+}
diff --git a/test/process_activefailover_default_test.go b/test/process_activefailover_default_test.go
index 7fbb00a8..4410dd84 100644
--- a/test/process_activefailover_default_test.go
+++ b/test/process_activefailover_default_test.go
@@ -96,10 +96,8 @@ func TestProcessActiveFailoverDefaultShutdownViaAPI(t *testing.T) {
 		testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 }
diff --git a/test/process_cluster_default_test.go b/test/process_cluster_default_test.go
index d315b58f..fca46258 100644
--- a/test/process_cluster_default_test.go
+++ b/test/process_cluster_default_test.go
@@ -94,12 +94,10 @@ func TestProcessClusterDefaultShutdownViaAPI(t *testing.T) {
 		testCluster(t, insecureStarterEndpoint(2*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 }
 
 // TestOldProcessClusterDefault starts a master starter, followed by 2 slave starters.
diff --git a/test/process_cluster_local_test.go b/test/process_cluster_local_test.go
index 3b6be68a..793344d9 100644
--- a/test/process_cluster_local_test.go
+++ b/test/process_cluster_local_test.go
@@ -74,10 +74,8 @@ func TestProcessClusterLocalShutdownViaAPI(t *testing.T) {
 		testCluster(t, insecureStarterEndpoint(2*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)))
 }
 
 // TestOldProcessClusterLocal runs `arangodb --local`
diff --git a/test/process_cluster_resign_leadership_test.go b/test/process_cluster_resign_leadership_test.go
index 8803e4de..06c4dcfb 100644
--- a/test/process_cluster_resign_leadership_test.go
+++ b/test/process_cluster_resign_leadership_test.go
@@ -163,7 +163,9 @@ func TestProcessClusterResignLeadership(t *testing.T) {
 		}
 	}()
 
-	ShutdownStarter(t, starterEndpointWithLeader)
+	waitForCallFunction(t,
+		ShutdownStarterCall(starterEndpointWithLeader))
+
 	cancel()
 	wg.Wait()
 	if errRead != nil {
@@ -185,7 +187,8 @@ func TestProcessClusterResignLeadership(t *testing.T) {
 		if endpoint == starterEndpointWithLeader {
 			continue
 		}
-		ShutdownStarter(t, endpoint)
+		waitForCallFunction(t,
+			ShutdownStarterCall(endpoint))
 	}
 }
 
diff --git a/test/process_resilientsingle_default_test.go b/test/process_resilientsingle_default_test.go
index 6d5997ef..eed914a3 100644
--- a/test/process_resilientsingle_default_test.go
+++ b/test/process_resilientsingle_default_test.go
@@ -96,10 +96,8 @@ func TestProcessResilientSingleDefaultShutdownViaAPI(t *testing.T) {
 		testResilientSingle(t, insecureStarterEndpoint(2*portIncrement), false, false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
-	ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(1*portIncrement)),
+		ShutdownStarterCall(insecureStarterEndpoint(2*portIncrement)))
 }
diff --git a/test/process_single_test.go b/test/process_single_test.go
index 3f9bd597..4c8e8754 100644
--- a/test/process_single_test.go
+++ b/test/process_single_test.go
@@ -70,10 +70,8 @@ func TestProcessSingleShutdownViaAPI(t *testing.T) {
 		testSingle(t, insecureStarterEndpoint(0*portIncrement), false)
 	}
 
-	if isVerbose {
-		t.Log("Waiting for termination")
-	}
-	ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
+	waitForCallFunction(t,
+		ShutdownStarterCall(insecureStarterEndpoint(0*portIncrement)))
 }
 
 // TestProcessSingleAutoKeyFile runs `arangodb --starter.mode=single --ssl.auto-key`
diff --git a/test/util.go b/test/util.go
index 5bf96eb2..2fd740f5 100644
--- a/test/util.go
+++ b/test/util.go
@@ -144,53 +144,41 @@ func SetUniqueDataDir(t *testing.T) string {
 	return dataDir
 }
 
-type waitUntilReadyResult struct {
-	Ready    bool
-	TimeSpan time.Duration
-	Message  string
-}
-
 // WaitUntilStarterReady waits until all given starter processes have reached the "Your cluster is ready state"
 func WaitUntilStarterReady(t *testing.T, what string, requiredGoodResults int, starters ...*SubProcess) bool {
-	results := make(chan waitUntilReadyResult, len(starters))
+	results := make([]error, len(starters))
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	for index, starter := range starters {
-		starter := starter // Used in nested function
-		id := fmt.Sprintf("starter-%d", index+1)
-		go func() {
-			started := time.Now()
-			if err := starter.ExpectTimeout(ctx, time.Minute*3, regexp.MustCompile(fmt.Sprintf("Your %s can now be accessed with a browser at", what)), id); err != nil {
-				timeSpan := time.Since(started)
-				results <- waitUntilReadyResult{
-					Ready:    false,
-					TimeSpan: timeSpan,
-					Message:  fmt.Sprintf("Starter is not ready in time (after %s): %s", timeSpan, describe(err)),
-				}
-			} else {
-				results <- waitUntilReadyResult{
-					Ready: true,
-				}
-			}
-		}()
+
+	var wg sync.WaitGroup
+
+	wg.Add(len(results))
+
+	for id, starter := range starters {
+		go func(i int, s *SubProcess) {
+			defer wg.Done()
+			defer cancel()
+			id := fmt.Sprintf("starter-%d", i+1)
+
+			results[i] = s.ExpectTimeout(ctx, time.Minute*3, regexp.MustCompile(fmt.Sprintf("Your %s can now be accessed with a browser at", what)), id)
+		}(id, starter)
 	}
-	okCount := 0
-	errorCount := 0
-	errorMessages := make([]string, 0, len(starters))
-	for result := range results {
-		if result.Ready {
-			okCount++
-		} else {
-			errorCount++
-			errorMessages = append(errorMessages, result.Message)
-		}
-		if okCount >= requiredGoodResults {
-			return true
-		}
-		if okCount+errorCount == len(starters) {
-			break
+
+	wg.Wait()
+
+	failed := 0
+	for _, result := range results {
+		if result != nil {
+			failed++
 		}
 	}
+
+	if failed <= requiredGoodResults {
+		GetLogger(t).Log("Starter Started")
+		return true
+	}
+
 	if os.Getenv("DEBUG_CLUSTER") == "interactive" {
 		// Halt forever
 		fmt.Println("Cluster not ready in time, halting forever for debugging")
@@ -198,9 +186,10 @@ func WaitUntilStarterReady(t *testing.T, what string, requiredGoodResults int, s
 			time.Sleep(time.Hour)
 		}
 	}
-	for _, msg := range errorMessages {
+	for _, msg := range results {
 		t.Error(msg)
 	}
+
 	return false
 }
 
@@ -257,11 +246,26 @@ func NewStarterClient(t *testing.T, endpoint string) client.API {
 	return c
 }
 
-// ShutdownStarter calls the starter the shutdown via the HTTP API.
-func ShutdownStarter(t *testing.T, endpoint string) {
+// ShutdownStarterCall returns function representation of ShutdownStarter.
+func ShutdownStarterCall(endpoint string) callFunction {
+	return func(t *testing.T) {
+		shutdownStarter(t, endpoint)
+	}
+}
+
+// shutdownStarter calls the starter the shutdown via the HTTP API.
+func shutdownStarter(t *testing.T, endpoint string) {
+	log := GetLogger(t)
+
+	log.Log("Terminating %s", endpoint)
+
+	defer func() {
+		log.Log("Terminated %s", endpoint)
+	}()
+
 	c := NewStarterClient(t, endpoint)
 	if err := c.Shutdown(context.Background(), false); err != nil {
-		t.Errorf("Shutdown failed: %s", describe(err))
+		log.Log("Shutdown failed: %s", describe(err))
 	}
 	WaitUntilStarterGone(t, endpoint)
 }
@@ -301,3 +305,20 @@ func createLicenseKeyOption() string {
 	}
 	return ""
 }
+
+type callFunction func(t *testing.T)
+
+func waitForCallFunction(t *testing.T, funcs ...callFunction) {
+	var wg sync.WaitGroup
+
+	wg.Add(len(funcs))
+
+	for _, f := range funcs {
+		go func(z callFunction) {
+			defer wg.Done()
+			z(t)
+		}(f)
+	}
+
+	wg.Wait()
+}

From cb020fbfcf55b15e074e0a4e05b37676550ccaba Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 12:24:55 +0000
Subject: [PATCH 07/37] Add failfast option

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index cf6348bc..006e19fd 100644
--- a/Makefile
+++ b/Makefile
@@ -198,7 +198,7 @@ run-tests-local-process-run:
 	$(DOCKER_CMD) /usr/code/bin/linux/amd64/test -test.timeout $(TEST_TIMEOUT) -test.v $(TESTOPTIONS)
 
 _run-tests: build-test build
-	@TEST_MODES=$(TEST_MODES) STARTER_MODES=$(STARTER_MODES) STARTER=$(BIN) ENTERPRISE=$(ENTERPRISE) IP=$(IP) ARANGODB=$(ARANGODB) $(TESTBIN) -test.timeout $(TEST_TIMEOUT) -test.v $(TESTOPTIONS)
+	@TEST_MODES=$(TEST_MODES) STARTER_MODES=$(STARTER_MODES) STARTER=$(BIN) ENTERPRISE=$(ENTERPRISE) IP=$(IP) ARANGODB=$(ARANGODB) $(TESTBIN) -test.timeout $(TEST_TIMEOUT) -test.failfast -test.v $(TESTOPTIONS)
 
 ifdef TRAVIS
 run-tests-docker-pre: docker

From 33c722b2176f842dea152cbf7aed3f102ddb018e Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 12:52:41 +0000
Subject: [PATCH 08/37] Add logger to docker removal

---
 test/docker_cluster_recovery_test.go | 10 +++---
 test/docker_util.go                  | 46 ++++++++++++++++++++++++++--
 test/gexpect.go                      | 14 +++++++++
 3 files changed, 62 insertions(+), 8 deletions(-)

diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index 4a54cd5e..c63770aa 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -34,6 +34,8 @@ import (
 // TestDockerClusterRecovery starts a master starter in docker, followed by 2 slave starters.
 // Once started, it destroys one of the starters and attempts a recovery.
 func TestDockerClusterRecovery(t *testing.T) {
+	log := GetLogger(t)
+
 	needTestMode(t, testModeDocker)
 	needStarterMode(t, starterModeCluster)
 	if os.Getenv("IP") == "" {
@@ -132,9 +134,7 @@ func TestDockerClusterRecovery(t *testing.T) {
 		testCluster(t, insecureStarterEndpoint(200), false)
 	}
 
-	if isVerbose {
-		t.Log("Start killing slave1 and its servers")
-	}
+	log.Log("Kill Server1")
 
 	// Cluster is up.
 	// Kill starter slave-1 and all its processes
@@ -156,9 +156,7 @@ func TestDockerClusterRecovery(t *testing.T) {
 	// Remove entire docker volume
 	removeDockerVolume(t, volID2)
 
-	if isVerbose {
-		t.Log("Starting recovery...")
-	}
+	log.Log("Recovery")
 
 	// Create new volume
 	recVolID2 := createDockerID("vol-starter-test-cluster-recovery2-recovery-")
diff --git a/test/docker_util.go b/test/docker_util.go
index 7563e5e3..fd9733f6 100644
--- a/test/docker_util.go
+++ b/test/docker_util.go
@@ -23,6 +23,8 @@
 package test
 
 import (
+	"bufio"
+	"bytes"
 	"crypto/rand"
 	"encoding/hex"
 	"fmt"
@@ -44,15 +46,55 @@ func removeDockerVolume(t *testing.T, id string) {
 }
 
 func removeDockerContainer(t *testing.T, id string) {
+	if t.Failed() {
+		logDockerPS(t)
+		logDockerLogs(t, id)
+	}
+
 	c := Spawn(t, fmt.Sprintf("docker rm -f -v %s", id))
 	defer c.Close()
 	c.Wait()
 }
 
-func stopDockerContainer(t *testing.T, id string) {
-	c := Spawn(t, fmt.Sprintf("docker stop --time=120 %s", id))
+func logDockerPS(t *testing.T) {
+	log := GetLogger(t)
+
+	// Dump of logs if failed
+	c := Spawn(t, fmt.Sprintf("docker ps -a"))
+	defer c.Close()
+	c.Wait()
+
+	logProcessOutput(log, c, "List of containers: ")
+}
+
+func logDockerLogs(t *testing.T, id string) {
+	log := GetLogger(t)
+
+	// Dump of logs if failed
+	c := Spawn(t, fmt.Sprintf("docker logs --timestamps %s", id))
 	defer c.Close()
 	c.Wait()
+
+	logProcessOutput(log, c, "Log of container %s: ")
+}
+
+func logProcessOutput(log Logger, p *SubProcess, prefix string, args ...interface{}) {
+	pre := ""
+	if prefix != "" {
+		pre = fmt.Sprintf(prefix, args...)
+	}
+	for {
+		line, _, err := bufio.NewReader(bytes.NewReader(p.Output())).ReadLine()
+		if err != nil {
+			break
+		}
+
+		if pre != "" {
+			log.Log(string(line))
+		} else {
+			log.Log("%s%s", pre, string(line))
+		}
+	}
 }
 
 func removeDockerContainersByLabel(t *testing.T, labelKeyValue string) {
diff --git a/test/gexpect.go b/test/gexpect.go
index 22346d61..62e85fee 100644
--- a/test/gexpect.go
+++ b/test/gexpect.go
@@ -150,6 +150,20 @@ func (sp *SubProcess) Wait() error {
 	return nil
 }
 
+// Output get current output
+func (sp *SubProcess) Output() []byte {
+	sp.mutex.Lock()
+	defer sp.mutex.Unlock()
+
+	d := sp.output.Bytes()
+
+	r := make([]byte, len(d))
+
+	copy(r, d)
+
+	return r
+}
+
 // ExpectTimeout waits for the output of the process to match the given expression, or until a timeout occurs.
 // If a match on the given expression is found, the process output is discard until the end of the match and
 // nil is returned, otherwise a timeout error is returned.

From 519e96df2195ab3c863a14c89a0478a05f8fe452 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 14:45:19 +0000
Subject: [PATCH 09/37] Add logger to docker removal

---
 .travis.yml         | 16 +++++++++-------
 test/docker_util.go |  2 +-
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 01f53ef3..f4c3c7b4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,14 +9,16 @@ go:
   - "1.13.x"
 
 env:
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single" PULL_REQUEST=false
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single" PULL_REQUEST=false
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
 
-script: make $TEST_SUITE
+script: 'if [ "$TRAVIS_PULL_REQUEST" != "$PULL_REQUEST" ]; then make $TEST_SUITE; fi'
 
 # Install Docker CE
 before_install:
diff --git a/test/docker_util.go b/test/docker_util.go
index fd9733f6..b375f410 100644
--- a/test/docker_util.go
+++ b/test/docker_util.go
@@ -75,7 +75,7 @@ func logDockerLogs(t *testing.T, id string) {
 	defer c.Close()
 	c.Wait()
 
-	logProcessOutput(log, c, "Log of container %s: ")
+	logProcessOutput(log, c, "Log of container %s: ", id)
 }
 
 func logProcessOutput(log Logger, p *SubProcess, prefix string, args ...interface{}) {

From 3a4dec28d2499f9d2f440e645df7853cd93b2e85 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 14:59:46 +0000
Subject: [PATCH 10/37] Add logger to docker removal

---
 test/docker_util.go | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/test/docker_util.go b/test/docker_util.go
index b375f410..c051bb39 100644
--- a/test/docker_util.go
+++ b/test/docker_util.go
@@ -83,17 +83,21 @@ func logProcessOutput(log Logger, p *SubProcess, prefix string, args ...interfac
 	if prefix != "" {
 		pre = fmt.Sprintf(prefix, args...)
 	}
+
+	reader :=  bufio.NewReader(bytes.NewReader(p.Output()))
+
 	for {
-		line, _, err := bufio.NewReader(bytes.NewReader(p.Output())).ReadLine()
+		line, _, err := reader.ReadLine()
+		if len(line) > 0 {
+			if pre != "" {
+				log.Log(string(line))
+			} else {
+				log.Log("%s%s", pre, string(line))
+			}
+		}
 		if err != nil {
 			break
 		}
-
-		if pre != "" {
-			log.Log(string(line))
-		} else {
-			log.Log("%s%s", pre, string(line))
-		}
 	}
 }
 

From db271d42a3fb2f91a4139f3f81e5eadbc19b2276 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 19:26:23 +0000
Subject: [PATCH 11/37] Add logger to docker removal

---
 test/docker_cluster_recovery_test.go | 5 -----
 test/util.go                         | 8 ++++++++
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index c63770aa..cb78f3de 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -73,7 +73,6 @@ func TestDockerClusterRecovery(t *testing.T) {
 		"docker run -i",
 		"--label starter-test=true",
 		"--name=" + cID1,
-		"--rm",
 		createLicenseKeyOption(),
 		fmt.Sprintf("-p %d:%d", basePort, basePort),
 		fmt.Sprintf("-v %s:/data", volID1),
@@ -92,7 +91,6 @@ func TestDockerClusterRecovery(t *testing.T) {
 		"docker run -i",
 		"--label starter-test=true",
 		"--name=" + cID2,
-		"--rm",
 		createLicenseKeyOption(),
 		fmt.Sprintf("-p %d:%d", basePort+100, basePort+100),
 		fmt.Sprintf("-v %s:/data", volID2),
@@ -112,7 +110,6 @@ func TestDockerClusterRecovery(t *testing.T) {
 		"docker run -i",
 		"--label starter-test=true",
 		"--name=" + cID3,
-		"--rm",
 		createLicenseKeyOption(),
 		fmt.Sprintf("-p %d:%d", basePort+200, basePort+200),
 		fmt.Sprintf("-v %s:/data", volID3),
@@ -169,7 +166,6 @@ func TestDockerClusterRecovery(t *testing.T) {
 		"docker run -i",
 		"--label starter-test=true",
 		"--name=" + cID2 + "recovery-builder",
-		"--rm",
 		fmt.Sprintf("-v %s:/data", recVolID2),
 		"alpine",
 		fmt.Sprintf("sh -c \"echo %s > /data/RECOVERY\"", recoveryContent),
@@ -182,7 +178,6 @@ func TestDockerClusterRecovery(t *testing.T) {
 		"docker run -i",
 		"--label starter-test=true",
 		"--name=" + recCID2,
-		"--rm",
 		createLicenseKeyOption(),
 		fmt.Sprintf("-p %d:%d", basePort+100, basePort+100),
 		fmt.Sprintf("-v %s:/data", recVolID2),
diff --git a/test/util.go b/test/util.go
index 2fd740f5..5190e3c1 100644
--- a/test/util.go
+++ b/test/util.go
@@ -26,6 +26,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/arangodb/go-driver"
 	"io/ioutil"
 	"net/url"
 	"os"
@@ -193,6 +194,13 @@ func WaitUntilStarterReady(t *testing.T, what string, requiredGoodResults int, s
 	return false
 }
 
+type ServiceReadyCheck func (err error) bool
+
+// WaitUntilServiceReady retry on service code 503
+func WaitUntilServiceReady(t *testing.T, c driver.Client, checks ... ServiceReadyCheck) bool {
+
+}
+
 // SendIntrAndWait stops all all given starter processes by sending a Ctrl-C into it.
 // It then waits until the process has terminated.
 func SendIntrAndWait(t *testing.T, starters ...*SubProcess) bool {

From 74b2e546a1ac3068bcef283b0084001dc1eb2310 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 19:26:38 +0000
Subject: [PATCH 12/37] Add logger to docker removal

---
 test/util.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/util.go b/test/util.go
index 5190e3c1..e7870003 100644
--- a/test/util.go
+++ b/test/util.go
@@ -198,7 +198,7 @@ type ServiceReadyCheck func (err error) bool
 
 // WaitUntilServiceReady retry on service code 503
 func WaitUntilServiceReady(t *testing.T, c driver.Client, checks ... ServiceReadyCheck) bool {
-
+	return true
 }
 
 // SendIntrAndWait stops all all given starter processes by sending a Ctrl-C into it.

From 331589fd08027de6b443143d330634a8d8cdca87 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 19:29:11 +0000
Subject: [PATCH 13/37] Install only Docker CLI

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index f4c3c7b4..6b513551 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,4 +25,4 @@ before_install:
   - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
   - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
   - sudo apt-get update
-  - sudo apt-get -y install docker-ce
+  - sudo apt-get -y install docker-ce-cli

From 88cb7528089572a712f02933c504f15c55d20fca Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 19:31:20 +0000
Subject: [PATCH 14/37] Skip docker install

---
 .travis.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 6b513551..b218d828 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,9 +20,9 @@ env:
 
 script: 'if [ "$TRAVIS_PULL_REQUEST" != "$PULL_REQUEST" ]; then make $TEST_SUITE; fi'
 
-# Install Docker CE
-before_install:
-  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-  - sudo apt-get update
-  - sudo apt-get -y install docker-ce-cli
+## Install Docker CE
+#before_install:
+#  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+#  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+#  - sudo apt-get update
+#  - sudo apt-get -y install docker-ce-cli

From f265894ef6750f9d201394dd8ee9e65013df8aef Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 20:45:14 +0000
Subject: [PATCH 15/37] Wait for docker command to exit

---
 .travis.yml                          | 2 +-
 test/docker_cluster_recovery_test.go | 1 +
 test/gexpect.go                      | 7 +++++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index b218d828..c8e39d39 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,7 +18,7 @@ env:
   - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
   - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
 
-script: 'if [ "$TRAVIS_PULL_REQUEST" != "$PULL_REQUEST" ]; then make $TEST_SUITE; fi'
+script: 'if [ "$TRAVIS_PULL_REQUEST" == "$PULL_REQUEST" ]; then make $TEST_SUITE; fi'
 
 ## Install Docker CE
 #before_install:
diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index cb78f3de..d26afba3 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -149,6 +149,7 @@ func TestDockerClusterRecovery(t *testing.T) {
 	}
 	killDockerRun2 := Spawn(t, "docker rm -vf "+strings.Join(containersToKill, " "))
 	killDockerRun2.Wait()
+	dockerRun2.WaitT(t)
 
 	// Remove entire docker volume
 	removeDockerVolume(t, volID2)
diff --git a/test/gexpect.go b/test/gexpect.go
index 62e85fee..36524db7 100644
--- a/test/gexpect.go
+++ b/test/gexpect.go
@@ -26,12 +26,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/stretchr/testify/require"
 	"io"
 	"os"
 	"os/exec"
 	"regexp"
 	"sync"
 	"syscall"
+	"testing"
 	"time"
 
 	"github.com/pkg/errors"
@@ -150,6 +152,11 @@ func (sp *SubProcess) Wait() error {
 	return nil
 }
 
+// WaitT waits for the process to terminate with require.
+func (sp *SubProcess) WaitT(t *testing.T) {
+	require.NoError(t, sp.Wait())
+}
+
 // Output get current output
 func (sp *SubProcess) Output() []byte {
 	sp.mutex.Lock()

From 5075a729174746b6e7175659969bdfc0b0447d9d Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Wed, 9 Jun 2021 21:26:09 +0000
Subject: [PATCH 16/37] Wait for docker command to exit

---
 .travis.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c8e39d39..dd20cfd0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,14 +11,14 @@ go:
 env:
   - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single" PULL_REQUEST=false
   - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single" PULL_REQUEST=false
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" PULL_REQUEST=true
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
 
-script: 'if [ "$TRAVIS_PULL_REQUEST" == "$PULL_REQUEST" ]; then make $TEST_SUITE; fi'
+script: 'if [ "$TRAVIS_PULL_REQUEST" == "$PULL_REQUEST" ]; then env; make $TEST_SUITE; fi'
 
 ## Install Docker CE
 #before_install:

From 1529305a830d88d58eba2498228b163fe05d624c Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Thu, 10 Jun 2021 06:54:36 +0000
Subject: [PATCH 17/37] Wait for docker command to exit

---
 .travis.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index dd20cfd0..6642009a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,12 +13,12 @@ env:
   - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single" PULL_REQUEST=false
   - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
   - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+#  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
+#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
+#  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
 
-script: 'if [ "$TRAVIS_PULL_REQUEST" == "$PULL_REQUEST" ]; then env; make $TEST_SUITE; fi'
+script: 'env; if [ "$TRAVIS_PULL_REQUEST" == "$PULL_REQUEST" ]; then make $TEST_SUITE; fi'
 
 ## Install Docker CE
 #before_install:

From d0053a27e8be33e9f7496f6e0e3af624bd05cf2a Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Thu, 10 Jun 2021 07:59:23 +0000
Subject: [PATCH 18/37] Wait for docker command to exit

---
 .travis.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 6642009a..1951895e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,16 +9,16 @@ go:
   - "1.13.x"
 
 env:
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single" PULL_REQUEST=false
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single" PULL_REQUEST=false
-  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
-  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover"
-#  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover"
-#  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
-#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single"
+  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover" ON_PR=1
+  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6 STARTER_MODES="single,cluster,activefailover" ON_PR=1
+#  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover" ON_PR=1
+#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest STARTER_MODES="single,cluster,activefailover" ON_PR=1
+#  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" ON_PR=1
+#  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" ON_PR=1
 
-script: 'env; if [ "$TRAVIS_PULL_REQUEST" == "$PULL_REQUEST" ]; then make $TEST_SUITE; fi'
+script: 'if ([ "$TRAVIS_PULL_REQUEST" != "false" ] && [ "$ON_PR" != "" ]) || ([ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$ON_PR" == "" ]) ; then  make $TEST_SUITE; fi'
 
 ## Install Docker CE
 #before_install:

From b3a4f65f54da760641eb57509c3c69a82931e2f7 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Thu, 10 Jun 2021 09:02:43 +0000
Subject: [PATCH 19/37] Wait for docker command to exit

---
 test/docker_cluster_recovery_test.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index d26afba3..83f9d3c4 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -149,7 +149,9 @@ func TestDockerClusterRecovery(t *testing.T) {
 	}
 	killDockerRun2 := Spawn(t, "docker rm -vf "+strings.Join(containersToKill, " "))
 	killDockerRun2.Wait()
-	dockerRun2.WaitT(t)
+
+	// Wait for command to close
+	dockerRun2.Wait()
 
 	// Remove entire docker volume
 	removeDockerVolume(t, volID2)

From b1dca3c41071a5b161af41ca69e523364d3529b3 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Thu, 10 Jun 2021 09:34:00 +0000
Subject: [PATCH 20/37] Wait for docker command to exit

---
 go.mod                               |  6 +---
 go.sum                               |  7 ++++
 test/docker_cluster_recovery_test.go | 11 +++++-
 test/docker_util.go                  |  2 +-
 test/gexpect.go                      |  3 +-
 test/log.go                          |  9 +++++
 test/util.go                         | 53 +++++++++++++++++++++++++---
 7 files changed, 78 insertions(+), 13 deletions(-)

diff --git a/go.mod b/go.mod
index 92fe614d..e8282c0d 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ replace golang.org/x/sys => golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a
 require (
 	github.com/aktau/github-release v0.8.1
 	github.com/arangodb-helper/go-certificates v0.0.0-20180821055445-9fca24fc2680
-	github.com/arangodb/go-driver v0.0.0-20200624173407-d1c92a8bd2b8
+	github.com/arangodb/go-driver v0.0.0-20210608063820-cd7131034c24
 	github.com/arangodb/go-upgrade-rules v0.0.0-20200605091205-439fb1ee86e7
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/coreos/go-semver v0.3.0
@@ -17,7 +17,6 @@ require (
 	github.com/fatih/color v1.9.0
 	github.com/fsouza/go-dockerclient v1.6.5
 	github.com/github-release/github-release v0.8.1 // indirect
-	github.com/google/addlicense v0.0.0-20200622132530-df58acafd6d5 // indirect
 	github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/kevinburke/rest v0.0.0-20200429221318-0d2892b400f8 // indirect
@@ -33,8 +32,5 @@ require (
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
 	github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 // indirect
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
-	golang.org/x/mod v0.3.0 // indirect
-	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 // indirect
 	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect
-	golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347 // indirect
 )
diff --git a/go.sum b/go.sum
index 5703a27d..7ce2b599 100644
--- a/go.sum
+++ b/go.sum
@@ -17,6 +17,8 @@ github.com/arangodb-helper/go-certificates v0.0.0-20180821055445-9fca24fc2680 h1
 github.com/arangodb-helper/go-certificates v0.0.0-20180821055445-9fca24fc2680/go.mod h1:xDyzBwyYzcEhsaDXtmxCNM4p5BrtuoVYYsRTuJqmCeg=
 github.com/arangodb/go-driver v0.0.0-20200624173407-d1c92a8bd2b8 h1:kWyYp1Gtp4x458LYlk9uAoM5v0ifZX58frROb1pC7Jo=
 github.com/arangodb/go-driver v0.0.0-20200624173407-d1c92a8bd2b8/go.mod h1:JG79qtPYRxUB6CdGWSH1XwpolSBjthuZX+Iaz/H38rA=
+github.com/arangodb/go-driver v0.0.0-20210608063820-cd7131034c24 h1:Tmuxuw6asYMsGa73rB8vGcE4ipX+j1uBFsodYqtCJNE=
+github.com/arangodb/go-driver v0.0.0-20210608063820-cd7131034c24/go.mod h1:3NUekcRLpgheFIGEwcOvxilEW73MV1queNKW58k7sdc=
 github.com/arangodb/go-upgrade-rules v0.0.0-20200605091205-439fb1ee86e7 h1:zCY5fsv5apos+oAdd1bLr1UEFOHeIUDZCItbwU/u6XE=
 github.com/arangodb/go-upgrade-rules v0.0.0-20200605091205-439fb1ee86e7/go.mod h1:RkPIG6JJ2pcJUoymc18NxAJGraZd+iAEVnOTDjZey/w=
 github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g=
@@ -102,11 +104,13 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/addlicense v0.0.0-20200622132530-df58acafd6d5 h1:m6Z1Cm53o4VecQFxKCnvULGfIT0Igo3MX131i+00IIo=
 github.com/google/addlicense v0.0.0-20200622132530-df58acafd6d5/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA=
+github.com/google/addlicense v0.0.0-20200817051935-6f4cd4aacc89/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -235,6 +239,7 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -268,6 +273,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -297,6 +303,7 @@ golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347 h1:/e4fNMHdLn7SQSxTrRZTma2xjQW6ELdxcnpqMhpo9X4=
 golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200818005847-188abfa75333/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index 83f9d3c4..39c9ace7 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -147,16 +147,23 @@ func TestDockerClusterRecovery(t *testing.T) {
 	for _, s := range plist.Servers {
 		containersToKill = append(containersToKill, s.ContainerID)
 	}
+
+	checkpoint := log.Checkpoint()
+
+	checkpoint.Log("Kill docker containers")
+
 	killDockerRun2 := Spawn(t, "docker rm -vf "+strings.Join(containersToKill, " "))
 	killDockerRun2.Wait()
 
+	checkpoint.Log("Wait for docker command to stop")
+
 	// Wait for command to close
 	dockerRun2.Wait()
 
 	// Remove entire docker volume
 	removeDockerVolume(t, volID2)
 
-	log.Log("Recovery")
+	checkpoint.Log("Recovery")
 
 	// Create new volume
 	recVolID2 := createDockerID("vol-starter-test-cluster-recovery2-recovery-")
@@ -175,6 +182,7 @@ func TestDockerClusterRecovery(t *testing.T) {
 	}, " "))
 	dockerBuildRecoveryRun.Wait()
 
+	checkpoint.Log("Start docker container")
 	// Restart dockerRun2
 	recCID2 := createDockerID("starter-test-cluster-recovery2-recovery-")
 	recDockerRun2 := Spawn(t, strings.Join([]string{
@@ -194,6 +202,7 @@ func TestDockerClusterRecovery(t *testing.T) {
 	}, " "))
 	defer recDockerRun2.Close()
 	defer removeDockerContainer(t, recCID2)
+	checkpoint.Log("Docker container started")
 
 	// Wait until recovered
 	if ok := WaitUntilStarterReady(t, whatCluster, 1, recDockerRun2); ok {
diff --git a/test/docker_util.go b/test/docker_util.go
index c051bb39..066ce13d 100644
--- a/test/docker_util.go
+++ b/test/docker_util.go
@@ -84,7 +84,7 @@ func logProcessOutput(log Logger, p *SubProcess, prefix string, args ...interfac
 		pre = fmt.Sprintf(prefix, args...)
 	}
 
-	reader :=  bufio.NewReader(bytes.NewReader(p.Output()))
+	reader := bufio.NewReader(bytes.NewReader(p.Output()))
 
 	for {
 		line, _, err := reader.ReadLine()
diff --git a/test/gexpect.go b/test/gexpect.go
index 36524db7..16186a80 100644
--- a/test/gexpect.go
+++ b/test/gexpect.go
@@ -26,7 +26,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"github.com/stretchr/testify/require"
 	"io"
 	"os"
 	"os/exec"
@@ -36,6 +35,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/pkg/errors"
 )
 
diff --git a/test/log.go b/test/log.go
index cf5f7609..89e85948 100644
--- a/test/log.go
+++ b/test/log.go
@@ -63,6 +63,7 @@ type Logger interface {
 	Log(format string, args ...interface{})
 
 	SubLogger(t *testing.T) Logger
+	Checkpoint() Logger
 
 	Clean()
 }
@@ -74,6 +75,14 @@ type logger struct {
 	parent *logger
 }
 
+func (l *logger) Checkpoint() Logger {
+	return &logger{
+		start:  time.Now(),
+		t:      l.t,
+		parent: l,
+	}
+}
+
 func (l *logger) Clean() {
 	cleanLogger(l.t)
 }
diff --git a/test/util.go b/test/util.go
index e7870003..e87b23f1 100644
--- a/test/util.go
+++ b/test/util.go
@@ -26,8 +26,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/arangodb/go-driver"
 	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
 	"regexp"
@@ -36,6 +36,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/arangodb/go-driver"
+
 	"github.com/arangodb-helper/arangodb/client"
 	shell "github.com/kballard/go-shellquote"
 	"github.com/pkg/errors"
@@ -194,11 +196,52 @@ func WaitUntilStarterReady(t *testing.T, what string, requiredGoodResults int, s
 	return false
 }
 
-type ServiceReadyCheck func (err error) bool
+type ServiceReadyCheck func(t *testing.T, c driver.Client) bool
+
+// WaitUntilServiceReadyRetryOnError do not allow any errors to occur
+func WaitUntilServiceReadyRetryOnError(t *testing.T, c driver.Client) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	_, err := c.Version(ctx)
+	return err == nil
+}
+
+// WaitUntilServiceReadyRetryOn503 retry on 503 code from service
+func WaitUntilServiceReadyRetryOn503(t *testing.T, c driver.Client) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	_, err := c.Version(ctx)
+	if err == nil {
+		return true
+	}
+
+	if ae, ok := driver.AsArangoError(err); !ok {
+		// Ignore unknown errors
+		return true
+	} else {
+		// Check if 503 is returned
+		return ae.Code != http.StatusServiceUnavailable
+	}
+}
+
+// WaitUntilServiceReadyAPI return timeout function which waits until service is fully ready
+func WaitUntilServiceReadyAPI(t *testing.T, c driver.Client) TimeoutFunc {
+	return WaitUntilServiceReady(t, c, WaitUntilServiceReadyRetryOn503, WaitUntilServiceReadyRetryOnError)
+}
+
+// WaitUntilServiceReady retry on errors from service
+func WaitUntilServiceReady(t *testing.T, c driver.Client, checks ...ServiceReadyCheck) TimeoutFunc {
+	return func() error {
+		for _, check := range checks {
+			if !check(t, c) {
+				return nil
+			}
+		}
 
-// WaitUntilServiceReady retry on service code 503
-func WaitUntilServiceReady(t *testing.T, c driver.Client, checks ... ServiceReadyCheck) bool {
-	return true
+		return Interrupt{}
+	}
 }
 
 // SendIntrAndWait stops all all given starter processes by sending a Ctrl-C into it.

From 4fc98ab9a0af02b22432d0927ba7e0bda24f111b Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Thu, 10 Jun 2021 11:06:56 +0000
Subject: [PATCH 21/37] Wait for docker command to exit

---
 test/docker_cluster_recovery_test.go |  3 +++
 test/timeout.go                      | 19 +++++++++++++++++++
 test/util.go                         | 21 +++++++++++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index 39c9ace7..abf3693b 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -182,6 +182,9 @@ func TestDockerClusterRecovery(t *testing.T) {
 	}, " "))
 	dockerBuildRecoveryRun.Wait()
 
+	checkpoint.Log("Wait for port to be closed")
+	WaitForHttpPortClosed(checkpoint, NewThrottle(time.Second), recoveryContent).ExecuteT(t, time.Minute, time.Second)
+
 	checkpoint.Log("Start docker container")
 	// Restart dockerRun2
 	recCID2 := createDockerID("starter-test-cluster-recovery2-recovery-")
diff --git a/test/timeout.go b/test/timeout.go
index 987a1d01..b76ea43a 100644
--- a/test/timeout.go
+++ b/test/timeout.go
@@ -90,6 +90,25 @@ func (f TimeoutFunc) Execute(timeout, interval time.Duration) error {
 	}
 }
 
+func (f TimeoutFunc) ExecuteWithLog(log Logger, timeout, interval time.Duration) (err error) {
+	now := time.Now()
+
+	defer func() {
+		if err == nil {
+			log.Log("Success - took %s", time.Now().Sub(now).String())
+		} else {
+			log.Log("Error - took %s - %s", time.Now().Sub(now).String(), err.Error())
+		}
+	}()
+
+	err = f.Execute(timeout, interval)
+	return
+}
+
+func (f TimeoutFunc) ExecuteTWithLog(t *testing.T, log Logger, timeout, interval time.Duration) {
+	require.NoError(t, f.ExecuteWithLog(log, timeout, interval))
+}
+
 func (f TimeoutFunc) ExecuteT(t *testing.T, timeout, interval time.Duration) {
 	require.NoError(t, f.Execute(timeout, interval))
 }
diff --git a/test/util.go b/test/util.go
index e87b23f1..57830c3f 100644
--- a/test/util.go
+++ b/test/util.go
@@ -244,6 +244,27 @@ func WaitUntilServiceReady(t *testing.T, c driver.Client, checks ...ServiceReady
 	}
 }
 
+func WaitForHttpPortClosed(log Logger, throttle Throttle, url string) TimeoutFunc {
+	return func() error {
+		_, err := http.Get(url)
+		if err == nil {
+			throttle.Execute(func() {
+				log.Log("Got empty response")
+			})
+			return nil
+		}
+
+		if strings.Contains(err.Error(), "read: connection reset by peer") {
+			return NewInterrupt()
+		}
+
+		throttle.Execute(func() {
+			log.Log("Unknown error: %s", err.Error())
+		})
+		return nil
+	}
+}
+
 // SendIntrAndWait stops all all given starter processes by sending a Ctrl-C into it.
 // It then waits until the process has terminated.
 func SendIntrAndWait(t *testing.T, starters ...*SubProcess) bool {

From 1c585c900d58ca7d4e397440ecf3a6c585b5529a Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Thu, 10 Jun 2021 11:52:31 +0000
Subject: [PATCH 22/37] Wait for docker command to exit

---
 test/docker_cluster_recovery_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index abf3693b..4440458b 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -183,7 +183,7 @@ func TestDockerClusterRecovery(t *testing.T) {
 	dockerBuildRecoveryRun.Wait()
 
 	checkpoint.Log("Wait for port to be closed")
-	WaitForHttpPortClosed(checkpoint, NewThrottle(time.Second), recoveryContent).ExecuteT(t, time.Minute, time.Second)
+	WaitForHttpPortClosed(checkpoint, NewThrottle(time.Second), insecureStarterEndpoint(100)).ExecuteT(t, time.Minute, time.Second)
 
 	checkpoint.Log("Start docker container")
 	// Restart dockerRun2

From 27c9f501ea9956935278b11f8a07741d4a6df0d1 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Thu, 10 Jun 2021 12:37:07 +0000
Subject: [PATCH 23/37] Wait for docker command to exit

---
 test/util.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/util.go b/test/util.go
index 57830c3f..c18a4032 100644
--- a/test/util.go
+++ b/test/util.go
@@ -254,7 +254,7 @@ func WaitForHttpPortClosed(log Logger, throttle Throttle, url string) TimeoutFun
 			return nil
 		}
 
-		if strings.Contains(err.Error(), "read: connection reset by peer") {
+		if strings.Contains(err.Error(), "connection refused") {
 			return NewInterrupt()
 		}
 

From 59bb62fc1dce148ebf7d64e2c3493fd85fde0840 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Thu, 10 Jun 2021 13:16:41 +0000
Subject: [PATCH 24/37] Wait for docker command to exit

---
 test/server_util.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/server_util.go b/test/server_util.go
index 4b5a6936..b3b0c495 100644
--- a/test/server_util.go
+++ b/test/server_util.go
@@ -131,6 +131,9 @@ func testProcesses(t *testing.T, c client.API, mode, starterEndpoint string, isS
 
 	// Wait until starter restarts
 	waitForStarter(t, c)
+	log := GetLogger(t)
+
+	log.Log("Starter is responding: %s", starterEndpoint)
 
 	// Fetch version
 	if info, err := c.Version(ctx); err != nil {

From 7dd73e5d019493b0f1b1828233b41a2230cd1f6b Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Thu, 10 Jun 2021 18:12:04 +0000
Subject: [PATCH 25/37] Wait for docker command to exit

---
 test/docker_cluster_recovery_test.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index 4440458b..bfdc598c 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -193,6 +193,7 @@ func TestDockerClusterRecovery(t *testing.T) {
 		"--label starter-test=true",
 		"--name=" + recCID2,
 		createLicenseKeyOption(),
+		"--net=host",
 		fmt.Sprintf("-p %d:%d", basePort+100, basePort+100),
 		fmt.Sprintf("-v %s:/data", recVolID2),
 		"-v /var/run/docker.sock:/var/run/docker.sock",

From a9982978af308b4a136da6ba6d8d03327cffe73d Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Thu, 10 Jun 2021 18:53:43 +0000
Subject: [PATCH 26/37] Wait for docker command to exit

---
 test/docker_cluster_recovery_test.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index bfdc598c..6171abce 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -185,6 +185,9 @@ func TestDockerClusterRecovery(t *testing.T) {
 	checkpoint.Log("Wait for port to be closed")
 	WaitForHttpPortClosed(checkpoint, NewThrottle(time.Second), insecureStarterEndpoint(100)).ExecuteT(t, time.Minute, time.Second)
 
+	// Clean Docker iptables routing
+	time.Sleep(10*time.Second)
+
 	checkpoint.Log("Start docker container")
 	// Restart dockerRun2
 	recCID2 := createDockerID("starter-test-cluster-recovery2-recovery-")
@@ -193,7 +196,6 @@ func TestDockerClusterRecovery(t *testing.T) {
 		"--label starter-test=true",
 		"--name=" + recCID2,
 		createLicenseKeyOption(),
-		"--net=host",
 		fmt.Sprintf("-p %d:%d", basePort+100, basePort+100),
 		fmt.Sprintf("-v %s:/data", recVolID2),
 		"-v /var/run/docker.sock:/var/run/docker.sock",

From 02d7216eb3a11dc946b90530e955d894c7bcde2d Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Fri, 11 Jun 2021 07:32:25 +0000
Subject: [PATCH 27/37] Wait for docker command to exit

---
 test/docker_cluster_recovery_test.go | 1 -
 test/docker_util.go                  | 4 ++++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index 6171abce..6e514225 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -185,7 +185,6 @@ func TestDockerClusterRecovery(t *testing.T) {
 	checkpoint.Log("Wait for port to be closed")
 	WaitForHttpPortClosed(checkpoint, NewThrottle(time.Second), insecureStarterEndpoint(100)).ExecuteT(t, time.Minute, time.Second)
 
-	// Clean Docker iptables routing
 	time.Sleep(10*time.Second)
 
 	checkpoint.Log("Start docker container")
diff --git a/test/docker_util.go b/test/docker_util.go
index 066ce13d..b44d532b 100644
--- a/test/docker_util.go
+++ b/test/docker_util.go
@@ -31,6 +31,7 @@ import (
 	"os/exec"
 	"strings"
 	"testing"
+	"time"
 )
 
 func createDockerVolume(t *testing.T, id string) {
@@ -62,6 +63,9 @@ func logDockerPS(t *testing.T) {
 	// Dump of logs if failed
 	c := Spawn(t, fmt.Sprintf("docker ps -a"))
 	defer c.Close()
+
+	time.Sleep(500*time.Millisecond)
+
 	c.Wait()
 
 	logProcessOutput(log, c, "List of containers: ")

From 9652c8be6c6611257a62898cb30302e7e9960fe0 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Fri, 11 Jun 2021 07:35:56 +0000
Subject: [PATCH 28/37] Wait for docker command to exit

---
 test/docker_cluster_recovery_test.go |  4 +--
 test/docker_util.go                  |  2 +-
 test/util.go                         | 37 ++++++++++++++++++++--------
 3 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index 6e514225..46271fd0 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -36,6 +36,8 @@ import (
 func TestDockerClusterRecovery(t *testing.T) {
 	log := GetLogger(t)
 
+	SkipOnTravis(t, "Test does not work on TRAVIS VM")
+
 	needTestMode(t, testModeDocker)
 	needStarterMode(t, starterModeCluster)
 	if os.Getenv("IP") == "" {
@@ -185,8 +187,6 @@ func TestDockerClusterRecovery(t *testing.T) {
 	checkpoint.Log("Wait for port to be closed")
 	WaitForHttpPortClosed(checkpoint, NewThrottle(time.Second), insecureStarterEndpoint(100)).ExecuteT(t, time.Minute, time.Second)
 
-	time.Sleep(10*time.Second)
-
 	checkpoint.Log("Start docker container")
 	// Restart dockerRun2
 	recCID2 := createDockerID("starter-test-cluster-recovery2-recovery-")
diff --git a/test/docker_util.go b/test/docker_util.go
index b44d532b..2117c585 100644
--- a/test/docker_util.go
+++ b/test/docker_util.go
@@ -64,7 +64,7 @@ func logDockerPS(t *testing.T) {
 	c := Spawn(t, fmt.Sprintf("docker ps -a"))
 	defer c.Close()
 
-	time.Sleep(500*time.Millisecond)
+	time.Sleep(500 * time.Millisecond)
 
 	c.Wait()
 
diff --git a/test/util.go b/test/util.go
index c18a4032..cbbbf2c0 100644
--- a/test/util.go
+++ b/test/util.go
@@ -44,18 +44,35 @@ import (
 )
 
 const (
-	ctrlC                     = "\u0003"
-	whatCluster               = "cluster"
-	whatSingle                = "single server"
-	whatResilientSingle       = "resilient single server"
-	testModeProcess           = "localprocess"
-	testModeDocker            = "docker"
-	starterModeCluster        = "cluster"
-	starterModeSingle         = "single"
-	starterModeActiveFailover = "activefailover"
-	portIncrement             = 10
+	ctrlC                                         = "\u0003"
+	whatCluster                                   = "cluster"
+	whatSingle                                    = "single server"
+	whatResilientSingle                           = "resilient single server"
+	testModeProcess                               = "localprocess"
+	testModeDocker                                = "docker"
+	starterModeCluster                            = "cluster"
+	starterModeSingle                             = "single"
+	starterModeActiveFailover                     = "activefailover"
+	portIncrement                                 = 10
+	travisEnv                 EnvironmentVariable = "TRAVIS"
 )
 
+type EnvironmentVariable string
+
+func (e EnvironmentVariable) String() string {
+	return string(e)
+}
+
+func (e EnvironmentVariable) Lookup() (string, bool) {
+	return os.LookupEnv(e.String())
+}
+
+func SkipOnTravis(t *testing.T, format string, args ...interface{}) {
+	if _, ok := travisEnv.Lookup(); ok {
+		t.Skipf(format, args...)
+	}
+}
+
 var (
 	isVerbose    bool
 	isEnterprise bool

From 5e6177a83f18bfe0653f467a1a4de6e56bde4447 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Mon, 14 Jun 2021 10:17:29 +0000
Subject: [PATCH 29/37] Retry on 503

---
 test/docker_util.go                           | 25 -----------
 .../process_cluster_resign_leadership_test.go |  6 +++
 test/util.go                                  | 45 +++++++++++++++----
 3 files changed, 42 insertions(+), 34 deletions(-)

diff --git a/test/docker_util.go b/test/docker_util.go
index 2117c585..faa1f697 100644
--- a/test/docker_util.go
+++ b/test/docker_util.go
@@ -23,8 +23,6 @@
 package test
 
 import (
-	"bufio"
-	"bytes"
 	"crypto/rand"
 	"encoding/hex"
 	"fmt"
@@ -82,29 +80,6 @@ func logDockerLogs(t *testing.T, id string) {
 	logProcessOutput(log, c, "Log of container %s: ", id)
 }
 
-func logProcessOutput(log Logger, p *SubProcess, prefix string, args ...interface{}) {
-	pre := ""
-	if prefix != "" {
-		pre = fmt.Sprintf(prefix, args...)
-	}
-
-	reader := bufio.NewReader(bytes.NewReader(p.Output()))
-
-	for {
-		line, _, err := reader.ReadLine()
-		if len(line) > 0 {
-			if pre != "" {
-				log.Log(string(line))
-			} else {
-				log.Log("%s%s", pre, string(line))
-			}
-		}
-		if err != nil {
-			break
-		}
-	}
-}
-
 func removeDockerContainersByLabel(t *testing.T, labelKeyValue string) {
 	ps := exec.Command("docker", "ps", "-q", "--filter", "label="+labelKeyValue)
 	list, err := ps.Output()
diff --git a/test/process_cluster_resign_leadership_test.go b/test/process_cluster_resign_leadership_test.go
index 06c4dcfb..32aada26 100644
--- a/test/process_cluster_resign_leadership_test.go
+++ b/test/process_cluster_resign_leadership_test.go
@@ -80,6 +80,12 @@ func TestProcessClusterResignLeadership(t *testing.T) {
 
 	databaseName := "_system"
 	collectionName := "test"
+
+	WaitUntilServiceReadyAPI(t, coordinatorClient, func(t *testing.T, ctx context.Context, c driver.Client) error {
+		_, err := coordinatorClient.Database(context.Background(), databaseName)
+		return err
+	}).ExecuteT(t, 15*time.Second, 500*time.Millisecond)
+
 	database, err := coordinatorClient.Database(context.Background(), databaseName)
 	if err != nil {
 		t.Fatal(err.Error())
diff --git a/test/util.go b/test/util.go
index cbbbf2c0..a6a03e7e 100644
--- a/test/util.go
+++ b/test/util.go
@@ -23,6 +23,8 @@
 package test
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -213,23 +215,25 @@ func WaitUntilStarterReady(t *testing.T, what string, requiredGoodResults int, s
 	return false
 }
 
-type ServiceReadyCheck func(t *testing.T, c driver.Client) bool
+type ServiceReadyCheckFunc func(t *testing.T, ctx context.Context, c driver.Client) error
+type ServiceReadyCheck func(t *testing.T, c driver.Client, check ServiceReadyCheckFunc) bool
 
 // WaitUntilServiceReadyRetryOnError do not allow any errors to occur
-func WaitUntilServiceReadyRetryOnError(t *testing.T, c driver.Client) bool {
+func WaitUntilServiceReadyRetryOnError(t *testing.T, c driver.Client, check ServiceReadyCheckFunc) bool {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
-	_, err := c.Version(ctx)
+	err := check(t, ctx, c)
+
 	return err == nil
 }
 
 // WaitUntilServiceReadyRetryOn503 retry on 503 code from service
-func WaitUntilServiceReadyRetryOn503(t *testing.T, c driver.Client) bool {
+func WaitUntilServiceReadyRetryOn503(t *testing.T, c driver.Client, check ServiceReadyCheckFunc) bool {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
-	_, err := c.Version(ctx)
+	err := check(t, ctx, c)
 	if err == nil {
 		return true
 	}
@@ -244,15 +248,15 @@ func WaitUntilServiceReadyRetryOn503(t *testing.T, c driver.Client) bool {
 }
 
 // WaitUntilServiceReadyAPI return timeout function which waits until service is fully ready
-func WaitUntilServiceReadyAPI(t *testing.T, c driver.Client) TimeoutFunc {
-	return WaitUntilServiceReady(t, c, WaitUntilServiceReadyRetryOn503, WaitUntilServiceReadyRetryOnError)
+func WaitUntilServiceReadyAPI(t *testing.T, c driver.Client, check ServiceReadyCheckFunc) TimeoutFunc {
+	return WaitUntilServiceReady(t, c, check, WaitUntilServiceReadyRetryOn503, WaitUntilServiceReadyRetryOnError)
 }
 
 // WaitUntilServiceReady retry on errors from service
-func WaitUntilServiceReady(t *testing.T, c driver.Client, checks ...ServiceReadyCheck) TimeoutFunc {
+func WaitUntilServiceReady(t *testing.T, c driver.Client, checkFunc ServiceReadyCheckFunc, checks ...ServiceReadyCheck) TimeoutFunc {
 	return func() error {
 		for _, check := range checks {
-			if !check(t, c) {
+			if !check(t, c, checkFunc) {
 				return nil
 			}
 		}
@@ -411,3 +415,26 @@ func waitForCallFunction(t *testing.T, funcs ...callFunction) {
 
 	wg.Wait()
 }
+
+func logProcessOutput(log Logger, p *SubProcess, prefix string, args ...interface{}) {
+	pre := ""
+	if prefix != "" {
+		pre = fmt.Sprintf(prefix, args...)
+	}
+
+	reader := bufio.NewReader(bytes.NewReader(p.Output()))
+
+	for {
+		line, _, err := reader.ReadLine()
+		if len(line) > 0 {
+			if pre != "" {
+				log.Log(string(line))
+			} else {
+				log.Log("%s%s", pre, string(line))
+			}
+		}
+		if err != nil {
+			break
+		}
+	}
+}

From fae60860c2619e3ae0400f355ead77a405722bf8 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Mon, 14 Jun 2021 12:07:38 +0000
Subject: [PATCH 30/37] Retry on 503

---
 .../process_cluster_resign_leadership_test.go | 20 +++++++++++--------
 test/process_cluster_upgrade_test.go          |  3 +++
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/test/process_cluster_resign_leadership_test.go b/test/process_cluster_resign_leadership_test.go
index 32aada26..7fe5d13b 100644
--- a/test/process_cluster_resign_leadership_test.go
+++ b/test/process_cluster_resign_leadership_test.go
@@ -178,15 +178,19 @@ func TestProcessClusterResignLeadership(t *testing.T) {
 		t.Logf("Reading documents: %s", errRead.Error())
 	}
 
-	// check new leader of the shard.
-	newDBServerLeader, err := getServerIDLeaderForFirstShard(coordinatorClient, database, collectionName)
-	if err != nil {
-		t.Fatal(err.Error())
-	}
+	NewTimeoutFunc(func() error {
+		// check new leader of the shard.
+		newDBServerLeader, err := getServerIDLeaderForFirstShard(coordinatorClient, database, collectionName)
+		if err != nil {
+			return nil
+		}
 
-	if dbServerLeader == newDBServerLeader {
-		t.Fatalf("DB server's ID '%s' can not be the same after leadership resignation", dbServerLeader)
-	}
+		if dbServerLeader == newDBServerLeader {
+			return nil
+		}
+
+		return NewInterrupt()
+	}).ExecuteT(t, time.Minute, 500*time.Millisecond)
 
 	// close the rest of the starters.
 	for _, endpoint := range starterEndpoints {
diff --git a/test/process_cluster_upgrade_test.go b/test/process_cluster_upgrade_test.go
index 21326f6f..51c03991 100644
--- a/test/process_cluster_upgrade_test.go
+++ b/test/process_cluster_upgrade_test.go
@@ -72,6 +72,9 @@ func testUpgradeProcess(t *testing.T, endpoint string) {
 	t.Log("Starting database upgrade")
 	c := NewStarterClient(t, endpoint)
 	ctx := context.Background()
+
+	waitForStarter(t, c)
+
 	if err := c.StartDatabaseUpgrade(ctx, false); err != nil {
 		t.Fatalf("StartDatabaseUpgrade failed: %v", err)
 	}

From d6bec24962b17b6708626aa8f9b394256b0d9d4f Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Tue, 15 Jun 2021 06:10:21 +0000
Subject: [PATCH 31/37] Aww waiters

---
 test/process_cluster_recovery_test.go | 10 +++++++---
 test/process_cluster_upgrade_test.go  | 15 +++++++++++++++
 test/process_util.go                  | 10 ++++++++++
 3 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/test/process_cluster_recovery_test.go b/test/process_cluster_recovery_test.go
index 990f3113..4a8fd973 100644
--- a/test/process_cluster_recovery_test.go
+++ b/test/process_cluster_recovery_test.go
@@ -46,16 +46,19 @@ func TestProcessClusterRecovery(t *testing.T) {
 
 	master := Spawn(t, "${STARTER} --starter.port=8528 "+createEnvironmentStarterOptions())
 	defer master.Close()
+	defer printProcessLogs(t, master, "Master")
 
 	dataDirSlave1 := SetUniqueDataDir(t)
 	defer os.RemoveAll(dataDirSlave1)
 	slave1 := Spawn(t, "${STARTER} --starter.port=8628 --starter.join 127.0.0.1:8528 "+createEnvironmentStarterOptions())
 	defer slave1.Close()
+	defer printProcessLogs(t, slave1, "Slave1")
 
 	dataDirSlave2 := SetUniqueDataDir(t)
 	defer os.RemoveAll(dataDirSlave2)
 	slave2 := Spawn(t, "${STARTER} --starter.port=8728 --starter.join 127.0.0.1:8528 "+createEnvironmentStarterOptions())
 	defer slave2.Close()
+	defer printProcessLogs(t, slave2, "Slave2")
 
 	if ok := WaitUntilStarterReady(t, whatCluster, 3, master, slave1, slave2); ok {
 		t.Logf("Cluster start took %s", time.Since(start))
@@ -105,11 +108,12 @@ func TestProcessClusterRecovery(t *testing.T) {
 
 	// Restart slave1
 	os.Setenv("DATA_DIR", dataDirSlave1)
-	slave1 = Spawn(t, "${STARTER} --starter.port=8628 --starter.join 127.0.0.1:8528 "+createEnvironmentStarterOptions())
-	defer slave1.Close()
+	master = Spawn(t, "${STARTER} --starter.port=8628 --starter.join 127.0.0.1:8528 "+createEnvironmentStarterOptions())
+	defer master.Close()
+	defer printProcessLogs(t, master, "Master 2")
 
 	// Wait until recovered
-	if ok := WaitUntilStarterReady(t, whatCluster, 3, slave1); ok {
+	if ok := WaitUntilStarterReady(t, whatCluster, 3, master, slave1, slave2); ok {
 		t.Logf("Cluster start (with recovery) took %s", time.Since(start))
 		testCluster(t, insecureStarterEndpoint(0), false)
 		testCluster(t, insecureStarterEndpoint(100), false)
diff --git a/test/process_cluster_upgrade_test.go b/test/process_cluster_upgrade_test.go
index 51c03991..99f39374 100644
--- a/test/process_cluster_upgrade_test.go
+++ b/test/process_cluster_upgrade_test.go
@@ -27,6 +27,9 @@ import (
 	"os"
 	"testing"
 	"time"
+
+	"github.com/arangodb-helper/arangodb/client"
+	"github.com/arangodb/go-driver"
 )
 
 // TestProcessClusterUpgrade starts a master starter, followed by 2 slave starters.
@@ -75,6 +78,18 @@ func testUpgradeProcess(t *testing.T, endpoint string) {
 
 	waitForStarter(t, c)
 
+	auth := driver.BasicAuthentication("root", "")
+	starterEndpointForCoordinator := insecureStarterEndpoint(1 * portIncrement)
+	coordinatorClient, err := CreateClient(t, starterEndpointForCoordinator, client.ServerTypeCoordinator, auth)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	WaitUntilServiceReadyAPI(t, coordinatorClient, func(t *testing.T, ctx context.Context, c driver.Client) error {
+		_, err := coordinatorClient.Database(context.Background(), "_system")
+		return err
+	}).ExecuteT(t, 15*time.Second, 500*time.Millisecond)
+
 	if err := c.StartDatabaseUpgrade(ctx, false); err != nil {
 		t.Fatalf("StartDatabaseUpgrade failed: %v", err)
 	}
diff --git a/test/process_util.go b/test/process_util.go
index 8f5ac310..cce5f3cb 100644
--- a/test/process_util.go
+++ b/test/process_util.go
@@ -31,3 +31,13 @@ func removeArangodProcesses(t *testing.T) {
 	defer c.Close()
 	c.Wait()
 }
+
+func printProcessLogs(t *testing.T, s *SubProcess, name string) {
+	if !t.Failed() {
+		return
+	}
+
+	log := GetLogger(t)
+
+	logProcessOutput(log, s, "Log of process: ", name)
+}

From b693437d0f192c14e325697467683a4cfdd37eba Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Tue, 15 Jun 2021 06:21:53 +0000
Subject: [PATCH 32/37] Aww waiters

---
 test/process_util.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/process_util.go b/test/process_util.go
index cce5f3cb..d778d9d2 100644
--- a/test/process_util.go
+++ b/test/process_util.go
@@ -39,5 +39,5 @@ func printProcessLogs(t *testing.T, s *SubProcess, name string) {
 
 	log := GetLogger(t)
 
-	logProcessOutput(log, s, "Log of process: ", name)
+	logProcessOutput(log, s, "Log of process: %s", name)
 }

From 0b50ea00e945c0cf92a6601b571540be299d46b4 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Tue, 15 Jun 2021 07:28:34 +0000
Subject: [PATCH 33/37] Add waiters

---
 test/docker_cluster_recovery_test.go           | 2 +-
 test/process_cluster_recovery_test.go          | 2 ++
 test/process_cluster_resign_leadership_test.go | 9 +++++++--
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/test/docker_cluster_recovery_test.go b/test/docker_cluster_recovery_test.go
index 46271fd0..8e51854c 100644
--- a/test/docker_cluster_recovery_test.go
+++ b/test/docker_cluster_recovery_test.go
@@ -36,7 +36,7 @@ import (
 func TestDockerClusterRecovery(t *testing.T) {
 	log := GetLogger(t)
 
-	SkipOnTravis(t, "Test does not work on TRAVIS VM")
+	SkipOnTravis(t, "Test does not work on TRAVIS VM") // TODO: Fix needed
 
 	needTestMode(t, testModeDocker)
 	needStarterMode(t, starterModeCluster)
diff --git a/test/process_cluster_recovery_test.go b/test/process_cluster_recovery_test.go
index 4a8fd973..8f337e91 100644
--- a/test/process_cluster_recovery_test.go
+++ b/test/process_cluster_recovery_test.go
@@ -36,6 +36,8 @@ import (
 // TestProcessClusterRecovery starts a master starter, followed by 2 slave starters.
 // Once started, it destroys one of the starters and attempts a recovery.
 func TestProcessClusterRecovery(t *testing.T) {
+	SkipOnTravis(t, "Test does not work on TRAVIS VM") // TODO: Fix needed
+
 	removeArangodProcesses(t)
 	needTestMode(t, testModeProcess)
 	needStarterMode(t, starterModeCluster)
diff --git a/test/process_cluster_resign_leadership_test.go b/test/process_cluster_resign_leadership_test.go
index 7fe5d13b..0fb39cb8 100644
--- a/test/process_cluster_resign_leadership_test.go
+++ b/test/process_cluster_resign_leadership_test.go
@@ -38,6 +38,8 @@ import (
 // TestProcessClusterResignLeadership starts a master starter, followed by 2 slave starters.
 // It closes the starter where the leader of the shard resides and check whether new leader of the shard is elected.
 func TestProcessClusterResignLeadership(t *testing.T) {
+	log := GetLogger(t)
+
 	removeArangodProcesses(t)
 	needTestMode(t, testModeProcess)
 	needStarterMode(t, starterModeCluster)
@@ -169,8 +171,7 @@ func TestProcessClusterResignLeadership(t *testing.T) {
 		}
 	}()
 
-	waitForCallFunction(t,
-		ShutdownStarterCall(starterEndpointWithLeader))
+	waitForCallFunction(t, ShutdownStarterCall(starterEndpointWithLeader))
 
 	cancel()
 	wg.Wait()
@@ -178,14 +179,18 @@ func TestProcessClusterResignLeadership(t *testing.T) {
 		t.Logf("Reading documents: %s", errRead.Error())
 	}
 
+	log.Log("Waiting for shutdown of services")
+
 	NewTimeoutFunc(func() error {
 		// check new leader of the shard.
 		newDBServerLeader, err := getServerIDLeaderForFirstShard(coordinatorClient, database, collectionName)
 		if err != nil {
+			log.Log("Error while fetching shard details: %s", err.Error())
 			return nil
 		}
 
 		if dbServerLeader == newDBServerLeader {
+			log.Log("Shard leader is on same server %s", dbServerLeader)
 			return nil
 		}
 

From 727222fffebfdb5ebbbedde0af0affd5cae081e2 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Tue, 15 Jun 2021 10:57:20 +0000
Subject: [PATCH 34/37] Add waiters

---
 test/process_cluster_recovery_test.go | 12 ++++--------
 test/process_util.go                  |  8 +++++++-
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/test/process_cluster_recovery_test.go b/test/process_cluster_recovery_test.go
index 8f337e91..530689c4 100644
--- a/test/process_cluster_recovery_test.go
+++ b/test/process_cluster_recovery_test.go
@@ -47,20 +47,17 @@ func TestProcessClusterRecovery(t *testing.T) {
 	start := time.Now()
 
 	master := Spawn(t, "${STARTER} --starter.port=8528 "+createEnvironmentStarterOptions())
-	defer master.Close()
-	defer printProcessLogs(t, master, "Master")
+	defer closeProcess(t, master, "Master")
 
 	dataDirSlave1 := SetUniqueDataDir(t)
 	defer os.RemoveAll(dataDirSlave1)
 	slave1 := Spawn(t, "${STARTER} --starter.port=8628 --starter.join 127.0.0.1:8528 "+createEnvironmentStarterOptions())
-	defer slave1.Close()
-	defer printProcessLogs(t, slave1, "Slave1")
+	defer closeProcess(t, slave1, "Slave1")
 
 	dataDirSlave2 := SetUniqueDataDir(t)
 	defer os.RemoveAll(dataDirSlave2)
 	slave2 := Spawn(t, "${STARTER} --starter.port=8728 --starter.join 127.0.0.1:8528 "+createEnvironmentStarterOptions())
-	defer slave2.Close()
-	defer printProcessLogs(t, slave2, "Slave2")
+	defer closeProcess(t, slave2, "Slave2")
 
 	if ok := WaitUntilStarterReady(t, whatCluster, 3, master, slave1, slave2); ok {
 		t.Logf("Cluster start took %s", time.Since(start))
@@ -111,8 +108,7 @@ func TestProcessClusterRecovery(t *testing.T) {
 	// Restart slave1
 	os.Setenv("DATA_DIR", dataDirSlave1)
 	master = Spawn(t, "${STARTER} --starter.port=8628 --starter.join 127.0.0.1:8528 "+createEnvironmentStarterOptions())
-	defer master.Close()
-	defer printProcessLogs(t, master, "Master 2")
+	defer closeProcess(t, master, "Master 2")
 
 	// Wait until recovered
 	if ok := WaitUntilStarterReady(t, whatCluster, 3, master, slave1, slave2); ok {
diff --git a/test/process_util.go b/test/process_util.go
index d778d9d2..f48f8ef4 100644
--- a/test/process_util.go
+++ b/test/process_util.go
@@ -32,7 +32,13 @@ func removeArangodProcesses(t *testing.T) {
 	c.Wait()
 }
 
-func printProcessLogs(t *testing.T, s *SubProcess, name string) {
+func closeProcess(t *testing.T, s *SubProcess, name string) {
+	s.Close()
+
+	showProcessLogs(t, s, name)
+}
+
+func showProcessLogs(t *testing.T, s *SubProcess, name string) {
 	if !t.Failed() {
 		return
 	}

From 03161a2c27e9fda9789902ce08c62e1e9c817f6f Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Tue, 15 Jun 2021 12:22:48 +0000
Subject: [PATCH 35/37] Add waiters

---
 test/gexpect.go      | 11 ++++++++++-
 test/process_util.go |  8 ++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/test/gexpect.go b/test/gexpect.go
index 16186a80..f9cbb1f4 100644
--- a/test/gexpect.go
+++ b/test/gexpect.go
@@ -131,6 +131,7 @@ func (sp *SubProcess) SendIntr() error {
 // Kill the process after the given timeout.
 func (sp *SubProcess) WaitTimeout(timeout time.Duration) error {
 	done := make(chan struct{})
+	defer close(done)
 	go func() {
 		select {
 		case <-time.After(timeout):
@@ -141,7 +142,15 @@ func (sp *SubProcess) WaitTimeout(timeout time.Duration) error {
 		}
 	}()
 	err := sp.cmd.Wait()
-	close(done)
+
+	if err != nil {
+		if c, ok := err.(*os.SyscallError); ok {
+			if c.Syscall == "waitid" {
+				return nil
+			}
+		}
+	}
+
 	return maskAny(err)
 }
 
diff --git a/test/process_util.go b/test/process_util.go
index f48f8ef4..5903d15e 100644
--- a/test/process_util.go
+++ b/test/process_util.go
@@ -38,6 +38,14 @@ func closeProcess(t *testing.T, s *SubProcess, name string) {
 	showProcessLogs(t, s, name)
 }
 
+func listArangodProcesses(t *testing.T, log Logger) {
+	c := SpawnWithExpand(t, "pidof -x arangod", false)
+	defer c.Close()
+	c.Wait()
+
+	logProcessOutput(log, c, "Processes: ")
+}
+
 func showProcessLogs(t *testing.T, s *SubProcess, name string) {
 	if !t.Failed() {
 		return

From 13fa62d305c788d2061f1886db1945bc388c7fdb Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Tue, 15 Jun 2021 13:09:24 +0000
Subject: [PATCH 36/37] Add waiters

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 1951895e..d4a2f405 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,7 +18,7 @@ env:
 #  - VERBOSE=1 TEST_SUITE=run-tests-local-process GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" ON_PR=1
 #  - VERBOSE=1 TEST_SUITE=run-tests-docker        GOIMAGE=gcr.io/gcr-for-testing/golang:1.13.6 ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.11 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest STARTER_MODES="single,cluster,activefailover" ON_PR=1
 
-script: 'if ([ "$TRAVIS_PULL_REQUEST" != "false" ] && [ "$ON_PR" != "" ]) || ([ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$ON_PR" == "" ]) ; then  make $TEST_SUITE; fi'
+script: 'if ([ "$TRAVIS_PULL_REQUEST" != "false" ] && [ "$ON_PR" != "" ]) || ([ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$ON_PR" == "" ]) ; then env; make $TEST_SUITE; fi'
 
 ## Install Docker CE
 #before_install:

From f248c1be3682b4ac3c176d9458bc67097a387218 Mon Sep 17 00:00:00 2001
From: ajanikow <12255597+ajanikow@users.noreply.github.com>
Date: Tue, 15 Jun 2021 21:09:48 +0000
Subject: [PATCH 37/37] Add waiters

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 006e19fd..79db1407 100644
--- a/Makefile
+++ b/Makefile
@@ -103,6 +103,7 @@ DOCKER_CMD = $(DOCKERCLI) run \
                 -e GOOS=$(GOOS) \
                 -e GOARCH=$(GOARCH) \
                 -e CGO_ENABLED=0 \
+                -e TRAVIS=$(TRAVIS) \
                 $(DOCKER_PARAMS) \
                 -w /usr/code/ \
                 $(DOCKER_IMAGE)