From 85ff1f2211a4be35ee477eac09dbd29f9793c2ea Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:15:42 -0500
Subject: [PATCH 01/73] shim in start and stop methods

---
 test/e2e/pkg/infra/docker/docker.go |  3 +++
 test/e2e/pkg/infra/provider.go      | 13 ++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 04811aab5e1..4e1c1792e6c 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -2,6 +2,7 @@ package docker
 
 import (
 	"bytes"
+	"context"
 	"os"
 	"path/filepath"
 	"text/template"
@@ -32,6 +33,8 @@ func (p *Provider) Setup() error {
 	}
 	return nil
 }
+func (Provider) StartNode(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
+func (Provider) StopNode(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index 03b821de384..34cbb8be0dc 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -1,5 +1,11 @@
 package infra
 
+import (
+	"context"
+
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+)
+
 // Provider defines an API for manipulating the infrastructure of a
 // specific set of testnet infrastructure.
 type Provider interface {
@@ -7,6 +13,9 @@ type Provider interface {
 	// Setup generates any necessary configuration for the infrastructure
 	// provider during testnet setup.
 	Setup() error
+
+	StartNode(context.Context, *e2e.Node) error
+	StopNode(context.Context, *e2e.Node) error
 }
 
 // NoopProvider implements the provider interface by performing noops for every
@@ -15,6 +24,8 @@ type Provider interface {
 type NoopProvider struct {
 }
 
-func (NoopProvider) Setup() error { return nil }
+func (NoopProvider) Setup() error                                   { return nil }
+func (NoopProvider) StartNode(_ context.Context, _ *e2e.Node) error { return nil }
+func (NoopProvider) StopNode(_ context.Context, _ *e2e.Node) error  { return nil }
 
 var _ Provider = NoopProvider{}

From 5ef7449e181c80c97dcdaa656b4a6beb8af407fe Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:17:22 -0500
Subject: [PATCH 02/73] rename to 'tendermint'

---
 test/e2e/pkg/infra/docker/docker.go |  4 ++--
 test/e2e/pkg/infra/provider.go      | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 4e1c1792e6c..979b00b5b1d 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -33,8 +33,8 @@ func (p *Provider) Setup() error {
 	}
 	return nil
 }
-func (Provider) StartNode(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
-func (Provider) StopNode(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
+func (Provider) StartTendermint(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
+func (Provider) StopTendermint(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index 34cbb8be0dc..47fa7a702d6 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -14,8 +14,8 @@ type Provider interface {
 	// provider during testnet setup.
 	Setup() error
 
-	StartNode(context.Context, *e2e.Node) error
-	StopNode(context.Context, *e2e.Node) error
+	StartTendermint(context.Context, *e2e.Node) error
+	StopTendermint(context.Context, *e2e.Node) error
 }
 
 // NoopProvider implements the provider interface by performing noops for every
@@ -24,8 +24,8 @@ type Provider interface {
 type NoopProvider struct {
 }
 
-func (NoopProvider) Setup() error                                   { return nil }
-func (NoopProvider) StartNode(_ context.Context, _ *e2e.Node) error { return nil }
-func (NoopProvider) StopNode(_ context.Context, _ *e2e.Node) error  { return nil }
+func (NoopProvider) Setup() error                                         { return nil }
+func (NoopProvider) StartTendermint(_ context.Context, _ *e2e.Node) error { return nil }
+func (NoopProvider) StopTendermint(_ context.Context, _ *e2e.Node) error  { return nil }
 
 var _ Provider = NoopProvider{}

From 14bd151611c437de0dba0bd216a16fcce2780b03 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:19:15 -0500
Subject: [PATCH 03/73] rename to 'kill tendermint'

---
 test/e2e/pkg/infra/docker/docker.go | 2 +-
 test/e2e/pkg/infra/provider.go      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 979b00b5b1d..a6ccf89783b 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -34,7 +34,7 @@ func (p *Provider) Setup() error {
 	return nil
 }
 func (Provider) StartTendermint(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
-func (Provider) StopTendermint(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
+func (Provider) KillTendermint(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index 47fa7a702d6..1822d895ede 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -15,7 +15,7 @@ type Provider interface {
 	Setup() error
 
 	StartTendermint(context.Context, *e2e.Node) error
-	StopTendermint(context.Context, *e2e.Node) error
+	KillTendermint(context.Context, *e2e.Node) error
 }
 
 // NoopProvider implements the provider interface by performing noops for every
@@ -26,6 +26,6 @@ type NoopProvider struct {
 
 func (NoopProvider) Setup() error                                         { return nil }
 func (NoopProvider) StartTendermint(_ context.Context, _ *e2e.Node) error { return nil }
-func (NoopProvider) StopTendermint(_ context.Context, _ *e2e.Node) error  { return nil }
+func (NoopProvider) KillTendermint(_ context.Context, _ *e2e.Node) error  { return nil }
 
 var _ Provider = NoopProvider{}

From 1a84bf907e697af0853755b7744ea392504873c9 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:21:26 -0500
Subject: [PATCH 04/73] add terminate vs kill

---
 test/e2e/pkg/infra/docker/docker.go | 5 +++--
 test/e2e/pkg/infra/provider.go      | 8 +++++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index a6ccf89783b..b72dcfa764a 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -33,8 +33,9 @@ func (p *Provider) Setup() error {
 	}
 	return nil
 }
-func (Provider) StartTendermint(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
-func (Provider) KillTendermint(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
+func (Provider) StartTendermint(_ context.Context, _ *e2e.Node) error     { panic("unimplemented") }
+func (Provider) TerminateTendermint(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
+func (Provider) KillTendermint(_ context.Context, _ *e2e.Node) error      { panic("unimplemented") }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index 1822d895ede..bff50eb6d20 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -16,6 +16,7 @@ type Provider interface {
 
 	StartTendermint(context.Context, *e2e.Node) error
 	KillTendermint(context.Context, *e2e.Node) error
+	TerminateTendermint(context.Context, *e2e.Node) error
 }
 
 // NoopProvider implements the provider interface by performing noops for every
@@ -24,8 +25,9 @@ type Provider interface {
 type NoopProvider struct {
 }
 
-func (NoopProvider) Setup() error                                         { return nil }
-func (NoopProvider) StartTendermint(_ context.Context, _ *e2e.Node) error { return nil }
-func (NoopProvider) KillTendermint(_ context.Context, _ *e2e.Node) error  { return nil }
+func (NoopProvider) Setup() error                                             { return nil }
+func (NoopProvider) StartTendermint(_ context.Context, _ *e2e.Node) error     { return nil }
+func (NoopProvider) TerminateTendermint(_ context.Context, _ *e2e.Node) error { return nil }
+func (NoopProvider) KillTendermint(_ context.Context, _ *e2e.Node) error      { return nil }
 
 var _ Provider = NoopProvider{}

From dd572d44fbfaf8ddd4d28f2bb3149b206fc55024 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:34:56 -0500
Subject: [PATCH 05/73] add command package

---
 test/e2e/runner/cleanup.go |  6 ++++--
 test/e2e/runner/exec.go    | 34 ++++++----------------------------
 test/e2e/runner/test.go    |  4 +++-
 3 files changed, 13 insertions(+), 31 deletions(-)

diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go
index fbe7aa0b4cc..74e84f7fc72 100644
--- a/test/e2e/runner/cleanup.go
+++ b/test/e2e/runner/cleanup.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -8,6 +9,7 @@ import (
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
 )
 
 // Cleanup removes the Docker Compose containers and testnet directory.
@@ -32,13 +34,13 @@ func cleanupDocker() error {
 	// does this by default. Ugly, but works.
 	xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)`
 
-	err := exec("bash", "-c", fmt.Sprintf(
+	err := exec.Command(context.Background(), "bash", "-c", fmt.Sprintf(
 		"docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR))
 	if err != nil {
 		return err
 	}
 
-	err = exec("bash", "-c", fmt.Sprintf(
+	err = exec.Command(context.Background(), "bash", "-c", fmt.Sprintf(
 		"docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR))
 	if err != nil {
 		return err
diff --git a/test/e2e/runner/exec.go b/test/e2e/runner/exec.go
index e6e47ca0a96..eedc447336c 100644
--- a/test/e2e/runner/exec.go
+++ b/test/e2e/runner/exec.go
@@ -1,49 +1,27 @@
 package main
 
 import (
-	"fmt"
-	"os"
-	osexec "os/exec"
 	"path/filepath"
-)
-
-// execute executes a shell command.
-func exec(args ...string) error {
-	cmd := osexec.Command(args[0], args[1:]...) //nolint:gosec
-	out, err := cmd.CombinedOutput()
-	switch err := err.(type) {
-	case nil:
-		return nil
-	case *osexec.ExitError:
-		return fmt.Errorf("failed to run %q:\n%v", args, string(out))
-	default:
-		return err
-	}
-}
 
-// execVerbose executes a shell command while displaying its output.
-func execVerbose(args ...string) error {
-	cmd := osexec.Command(args[0], args[1:]...) //nolint:gosec
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
-}
+	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
+	"golang.org/x/net/context"
+)
 
 // execCompose runs a Docker Compose command for a testnet.
 func execCompose(dir string, args ...string) error {
-	return exec(append(
+	return exec.Command(context.Background(), append(
 		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
 		args...)...)
 }
 
 // execComposeVerbose runs a Docker Compose command for a testnet and displays its output.
 func execComposeVerbose(dir string, args ...string) error {
-	return execVerbose(append(
+	return exec.CommandVerbose(context.Background(), append(
 		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
 		args...)...)
 }
 
 // execDocker runs a Docker command.
 func execDocker(args ...string) error {
-	return exec(append([]string{"docker"}, args...)...)
+	return exec.Command(context.Background(), append([]string{"docker"}, args...)...)
 }
diff --git a/test/e2e/runner/test.go b/test/e2e/runner/test.go
index 834ce6f2d04..4671a81d262 100644
--- a/test/e2e/runner/test.go
+++ b/test/e2e/runner/test.go
@@ -1,9 +1,11 @@
 package main
 
 import (
+	"context"
 	"os"
 
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
 )
 
 // Test runs test cases under tests/
@@ -15,5 +17,5 @@ func Test(testnet *e2e.Testnet) error {
 		return err
 	}
 
-	return execVerbose("go", "test", "-count", "1", "./tests/...")
+	return exec.CommandVerbose(context.Background(), "go", "test", "-count", "1", "./tests/...")
 }

From b0f85035781e9563a1d638e5c24ce06990251094 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:38:26 -0500
Subject: [PATCH 06/73] add param names

---
 test/e2e/pkg/infra/docker/docker.go | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index b72dcfa764a..7ba8ca3cd4d 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -33,9 +33,15 @@ func (p *Provider) Setup() error {
 	}
 	return nil
 }
-func (Provider) StartTendermint(_ context.Context, _ *e2e.Node) error     { panic("unimplemented") }
-func (Provider) TerminateTendermint(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
-func (Provider) KillTendermint(_ context.Context, _ *e2e.Node) error      { panic("unimplemented") }
+func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
+	panic("unimplemented")
+}
+func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
+	panic("unimplemented")
+}
+func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
+	panic("unimplemented")
+}
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.

From b762c87ffa63552bdbfefef1bc3185a409eb4dbc Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:42:15 -0500
Subject: [PATCH 07/73] switch exec compose to docker pkg

---
 test/e2e/pkg/infra/docker/docker.go | 20 ++++++++++++++++++++
 test/e2e/runner/main.go             |  2 +-
 test/e2e/runner/perturb.go          | 11 ++++++-----
 test/e2e/runner/start.go            |  5 +++--
 4 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 7ba8ca3cd4d..f9d23927447 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -8,6 +8,7 @@ import (
 	"text/template"
 
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
 	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
 )
 
@@ -93,3 +94,22 @@ services:
 	}
 	return buf.Bytes(), nil
 }
+
+// ExecCompose runs a Docker Compose command for a testnet.
+func ExecCompose(dir string, args ...string) error {
+	return exec.Command(context.Background(), append(
+		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
+		args...)...)
+}
+
+// ExecComposeVerbose runs a Docker Compose command for a testnet and displays its output.
+func ExecComposeVerbose(dir string, args ...string) error {
+	return exec.CommandVerbose(context.Background(), append(
+		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
+		args...)...)
+}
+
+// ExecDocker runs a Docker command.
+func ExecDocker(args ...string) error {
+	return exec.Command(context.Background(), append([]string{"docker"}, args...)...)
+}
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index fdfec7bbed8..62322581317 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -211,7 +211,7 @@ func NewCLI() *CLI {
 		Short: "Stops the Docker testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			logger.Info("Stopping testnet")
-			return execCompose(cli.testnet.Dir, "down")
+			return docker.ExecCompose(cli.testnet.Dir, "down")
 		},
 	})
 
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index f47aa72bfc4..5180774d95e 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -7,6 +7,7 @@ import (
 	"github.com/tendermint/tendermint/libs/log"
 	rpctypes "github.com/tendermint/tendermint/rpc/core/types"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
 )
 
 // Perturbs a running testnet.
@@ -40,26 +41,26 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 
 	case e2e.PerturbationKill:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Killing node %v...", node.Name))
-		if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
 			return nil, err
 		}
-		if err := execCompose(testnet.Dir, "start", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "start", node.Name); err != nil {
 			return nil, err
 		}
 
 	case e2e.PerturbationPause:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Pausing node %v...", node.Name))
-		if err := execCompose(testnet.Dir, "pause", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "pause", node.Name); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "unpause", node.Name); err != nil {
 			return nil, err
 		}
 
 	case e2e.PerturbationRestart:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Restarting node %v...", node.Name))
-		if err := execCompose(testnet.Dir, "restart", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "restart", node.Name); err != nil {
 			return nil, err
 		}
 
diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go
index 0dab98f22d6..a3082e17d29 100644
--- a/test/e2e/runner/start.go
+++ b/test/e2e/runner/start.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
 )
 
 func Start(testnet *e2e.Testnet) error {
@@ -43,7 +44,7 @@ func Start(testnet *e2e.Testnet) error {
 	for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 {
 		node := nodeQueue[0]
 		nodeQueue = nodeQueue[1:]
-		if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}
 		if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
@@ -97,7 +98,7 @@ func Start(testnet *e2e.Testnet) error {
 
 		logger.Info("Starting catch up node", "node", node.Name, "height", node.StartAt)
 
-		if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}
 		status, err := waitForNode(node, node.StartAt, 3*time.Minute)

From c72f7c01b1fb26615e466a40b35f912b8f2d2220 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:43:52 -0500
Subject: [PATCH 08/73] all exec switched to docker package

---
 test/e2e/pkg/infra/docker/docker.go | 4 ++--
 test/e2e/runner/cleanup.go          | 3 ++-
 test/e2e/runner/main.go             | 4 ++--
 test/e2e/runner/perturb.go          | 4 ++--
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index f9d23927447..a74f7b5b53f 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -109,7 +109,7 @@ func ExecComposeVerbose(dir string, args ...string) error {
 		args...)...)
 }
 
-// ExecDocker runs a Docker command.
-func ExecDocker(args ...string) error {
+// Exec runs a Docker command.
+func Exec(args ...string) error {
 	return exec.Command(context.Background(), append([]string{"docker"}, args...)...)
 }
diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go
index 74e84f7fc72..5df402be80e 100644
--- a/test/e2e/runner/cleanup.go
+++ b/test/e2e/runner/cleanup.go
@@ -10,6 +10,7 @@ import (
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
 )
 
 // Cleanup removes the Docker Compose containers and testnet directory.
@@ -71,7 +72,7 @@ func cleanupDir(dir string) error {
 	if err != nil {
 		return err
 	}
-	err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
+	err = docker.Exec("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
 		"tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/")
 	if err != nil {
 		return err
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index 62322581317..ec5dfa94ac0 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -276,7 +276,7 @@ func NewCLI() *CLI {
 		Use:   "logs",
 		Short: "Shows the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return execComposeVerbose(cli.testnet.Dir, "logs")
+			return docker.ExecComposeVerbose(cli.testnet.Dir, "logs")
 		},
 	})
 
@@ -284,7 +284,7 @@ func NewCLI() *CLI {
 		Use:   "tail",
 		Short: "Tails the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return execComposeVerbose(cli.testnet.Dir, "logs", "--follow")
+			return docker.ExecComposeVerbose(cli.testnet.Dir, "logs", "--follow")
 		},
 	})
 
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index 5180774d95e..a08de233273 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -31,11 +31,11 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 	switch perturbation {
 	case e2e.PerturbationDisconnect:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Disconnecting node %v...", node.Name))
-		if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
+		if err := docker.Exec("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
+		if err := docker.Exec("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
 			return nil, err
 		}
 

From bd31f705719e65f19a664adc91689e621dc797d3 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:44:11 -0500
Subject: [PATCH 09/73] remove runner exec

---
 test/e2e/runner/exec.go | 27 ---------------------------
 1 file changed, 27 deletions(-)
 delete mode 100644 test/e2e/runner/exec.go

diff --git a/test/e2e/runner/exec.go b/test/e2e/runner/exec.go
deleted file mode 100644
index eedc447336c..00000000000
--- a/test/e2e/runner/exec.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package main
-
-import (
-	"path/filepath"
-
-	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
-	"golang.org/x/net/context"
-)
-
-// execCompose runs a Docker Compose command for a testnet.
-func execCompose(dir string, args ...string) error {
-	return exec.Command(context.Background(), append(
-		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
-		args...)...)
-}
-
-// execComposeVerbose runs a Docker Compose command for a testnet and displays its output.
-func execComposeVerbose(dir string, args ...string) error {
-	return exec.CommandVerbose(context.Background(), append(
-		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
-		args...)...)
-}
-
-// execDocker runs a Docker command.
-func execDocker(args ...string) error {
-	return exec.Command(context.Background(), append([]string{"docker"}, args...)...)
-}

From 91fa4f3296352b370bb245647ff50ea9b88c0b76 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:46:28 -0500
Subject: [PATCH 10/73] functionality in place for start kill terminate

---
 test/e2e/pkg/infra/docker/docker.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index a74f7b5b53f..24ffa675b52 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -35,13 +35,13 @@ func (p *Provider) Setup() error {
 	return nil
 }
 func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
-	panic("unimplemented")
+	return ExecCompose(p.Testnet.Dir, "start", n.Name)
 }
 func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
-	panic("unimplemented")
+	return ExecCompose(p.Testnet.Dir, "kill", "-s", "SIGTERM", n.Name)
 }
 func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
-	panic("unimplemented")
+	return ExecCompose(p.Testnet.Dir, "kill", "-s", "SIGKILL", n.Name)
 }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the

From 62e4fb0e7030bf8a64ecf572ae90e8de9076a25b Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:48:56 -0500
Subject: [PATCH 11/73] exec compose takes context

---
 test/e2e/pkg/infra/docker/docker.go | 10 +++++-----
 test/e2e/runner/main.go             |  2 +-
 test/e2e/runner/perturb.go          | 11 ++++++-----
 test/e2e/runner/start.go            |  5 +++--
 4 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 24ffa675b52..80798e26e09 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -35,13 +35,13 @@ func (p *Provider) Setup() error {
 	return nil
 }
 func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
-	return ExecCompose(p.Testnet.Dir, "start", n.Name)
+	return ExecCompose(ctx, p.Testnet.Dir, "start", n.Name)
 }
 func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
-	return ExecCompose(p.Testnet.Dir, "kill", "-s", "SIGTERM", n.Name)
+	return ExecCompose(ctx, p.Testnet.Dir, "kill", "-s", "SIGTERM", n.Name)
 }
 func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
-	return ExecCompose(p.Testnet.Dir, "kill", "-s", "SIGKILL", n.Name)
+	return ExecCompose(ctx, p.Testnet.Dir, "kill", "-s", "SIGKILL", n.Name)
 }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
@@ -96,8 +96,8 @@ services:
 }
 
 // ExecCompose runs a Docker Compose command for a testnet.
-func ExecCompose(dir string, args ...string) error {
-	return exec.Command(context.Background(), append(
+func ExecCompose(ctx context.Context, dir string, args ...string) error {
+	return exec.Command(ctx, append(
 		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
 		args...)...)
 }
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index ec5dfa94ac0..79279a1a459 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -211,7 +211,7 @@ func NewCLI() *CLI {
 		Short: "Stops the Docker testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			logger.Info("Stopping testnet")
-			return docker.ExecCompose(cli.testnet.Dir, "down")
+			return docker.ExecCompose(context.Background(), cli.testnet.Dir, "down") 
 		},
 	})
 
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index a08de233273..bab66680b5f 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -41,26 +42,26 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 
 	case e2e.PerturbationKill:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Killing node %v...", node.Name))
-		if err := docker.ExecCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
 			return nil, err
 		}
-		if err := docker.ExecCompose(testnet.Dir, "start", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "start", node.Name); err != nil {
 			return nil, err
 		}
 
 	case e2e.PerturbationPause:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Pausing node %v...", node.Name))
-		if err := docker.ExecCompose(testnet.Dir, "pause", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "pause", node.Name); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := docker.ExecCompose(testnet.Dir, "unpause", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "unpause", node.Name); err != nil {
 			return nil, err
 		}
 
 	case e2e.PerturbationRestart:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Restarting node %v...", node.Name))
-		if err := docker.ExecCompose(testnet.Dir, "restart", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "restart", node.Name); err != nil {
 			return nil, err
 		}
 
diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go
index a3082e17d29..08dab035f24 100644
--- a/test/e2e/runner/start.go
+++ b/test/e2e/runner/start.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"sort"
 	"time"
@@ -44,7 +45,7 @@ func Start(testnet *e2e.Testnet) error {
 	for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 {
 		node := nodeQueue[0]
 		nodeQueue = nodeQueue[1:]
-		if err := docker.ExecCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}
 		if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
@@ -98,7 +99,7 @@ func Start(testnet *e2e.Testnet) error {
 
 		logger.Info("Starting catch up node", "node", node.Name, "height", node.StartAt)
 
-		if err := docker.ExecCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}
 		status, err := waitForNode(node, node.StartAt, 3*time.Minute)

From b2740d39d878844f316d1ad9136ece2f01f4cdf2 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:49:56 -0500
Subject: [PATCH 12/73] compose verbose takes context

---
 test/e2e/pkg/infra/docker/docker.go | 4 ++--
 test/e2e/runner/main.go             | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 80798e26e09..19a2934aaff 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -103,8 +103,8 @@ func ExecCompose(ctx context.Context, dir string, args ...string) error {
 }
 
 // ExecComposeVerbose runs a Docker Compose command for a testnet and displays its output.
-func ExecComposeVerbose(dir string, args ...string) error {
-	return exec.CommandVerbose(context.Background(), append(
+func ExecComposeVerbose(ctx context.Context, dir string, args ...string) error {
+	return exec.CommandVerbose(ctx, append(
 		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
 		args...)...)
 }
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index 79279a1a459..195174b397b 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -276,7 +276,7 @@ func NewCLI() *CLI {
 		Use:   "logs",
 		Short: "Shows the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return docker.ExecComposeVerbose(cli.testnet.Dir, "logs")
+			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs") 
 		},
 	})
 
@@ -284,7 +284,7 @@ func NewCLI() *CLI {
 		Use:   "tail",
 		Short: "Tails the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return docker.ExecComposeVerbose(cli.testnet.Dir, "logs", "--follow")
+			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs", "--follow") 
 		},
 	})
 

From f0f98c8156c3a448931db4ab132ea6e2721aa71c Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:50:39 -0500
Subject: [PATCH 13/73] docker.exec takes context

---
 test/e2e/pkg/infra/docker/docker.go | 4 ++--
 test/e2e/runner/cleanup.go          | 4 ++--
 test/e2e/runner/perturb.go          | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 19a2934aaff..03c19cacfb1 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -110,6 +110,6 @@ func ExecComposeVerbose(ctx context.Context, dir string, args ...string) error {
 }
 
 // Exec runs a Docker command.
-func Exec(args ...string) error {
-	return exec.Command(context.Background(), append([]string{"docker"}, args...)...)
+func Exec(ctx context.Context, args ...string) error {
+	return exec.Command(ctx, append([]string{"docker"}, args...)...)
 }
diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go
index 5df402be80e..2cc7f9b68a5 100644
--- a/test/e2e/runner/cleanup.go
+++ b/test/e2e/runner/cleanup.go
@@ -72,8 +72,8 @@ func cleanupDir(dir string) error {
 	if err != nil {
 		return err
 	}
-	err = docker.Exec("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
-		"tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/")
+	err = docker.Exec(context.Background(), "run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
+		"tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") 
 	if err != nil {
 		return err
 	}
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index bab66680b5f..dfd0ff8b72b 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -32,11 +32,11 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 	switch perturbation {
 	case e2e.PerturbationDisconnect:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Disconnecting node %v...", node.Name))
-		if err := docker.Exec("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
+		if err := docker.Exec(context.Background(), "network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name) ; err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := docker.Exec("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
+		if err := docker.Exec(context.Background(), "network", "connect", testnet.Name+"_"+testnet.Name, node.Name) ; err != nil {
 			return nil, err
 		}
 

From 3a8ffa5219307a5d2507a0faa64eb15acd513e81 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:00:41 -0500
Subject: [PATCH 14/73] add create node to infra provider

---
 test/e2e/pkg/infra/docker/docker.go | 4 ++++
 test/e2e/pkg/infra/provider.go      | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 03c19cacfb1..a177002985b 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -34,6 +34,10 @@ func (p *Provider) Setup() error {
 	}
 	return nil
 }
+
+func (p Provider) CreateNode(ctx context.Context, n *e2e.Node) error {
+	return ExecCompose(ctx, p.Testnet.Dir, "create", n.Name)
+}
 func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
 	return ExecCompose(ctx, p.Testnet.Dir, "start", n.Name)
 }
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index bff50eb6d20..0fdcc86baa8 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -14,6 +14,7 @@ type Provider interface {
 	// provider during testnet setup.
 	Setup() error
 
+	CreateNode(context.Context, *e2e.Node) error
 	StartTendermint(context.Context, *e2e.Node) error
 	KillTendermint(context.Context, *e2e.Node) error
 	TerminateTendermint(context.Context, *e2e.Node) error
@@ -26,6 +27,7 @@ type NoopProvider struct {
 }
 
 func (NoopProvider) Setup() error                                             { return nil }
+func (NoopProvider) CreateNode(_ context.Context, _ *e2e.Node) error          { return nil }
 func (NoopProvider) StartTendermint(_ context.Context, _ *e2e.Node) error     { return nil }
 func (NoopProvider) TerminateTendermint(_ context.Context, _ *e2e.Node) error { return nil }
 func (NoopProvider) KillTendermint(_ context.Context, _ *e2e.Node) error      { return nil }

From 558268e4e869e572ee2ce5c61897fd1a510ca04c Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:06:30 -0500
Subject: [PATCH 15/73] start and create calls replaced with infra provider

---
 test/e2e/runner/main.go  | 12 ++++++------
 test/e2e/runner/start.go | 18 ++++++++++++++----
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index 195174b397b..f7541cc7386 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -112,7 +112,7 @@ func NewCLI() *CLI {
 				chLoadResult <- err
 			}()
 
-			if err := Start(cli.testnet); err != nil {
+			if err := Start(cli.testnet, cli.infp); err != nil {
 				return err
 			}
 
@@ -186,7 +186,7 @@ func NewCLI() *CLI {
 			if err != nil {
 				return err
 			}
-			return Start(cli.testnet)
+			return Start(cli.testnet, cli.infp)
 		},
 	})
 
@@ -211,7 +211,7 @@ func NewCLI() *CLI {
 		Short: "Stops the Docker testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			logger.Info("Stopping testnet")
-			return docker.ExecCompose(context.Background(), cli.testnet.Dir, "down") 
+			return docker.ExecCompose(context.Background(), cli.testnet.Dir, "down")
 		},
 	})
 
@@ -276,7 +276,7 @@ func NewCLI() *CLI {
 		Use:   "logs",
 		Short: "Shows the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs") 
+			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs")
 		},
 	})
 
@@ -284,7 +284,7 @@ func NewCLI() *CLI {
 		Use:   "tail",
 		Short: "Tails the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs", "--follow") 
+			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs", "--follow")
 		},
 	})
 
@@ -319,7 +319,7 @@ Does not run any perbutations.
 				chLoadResult <- err
 			}()
 
-			if err := Start(cli.testnet); err != nil {
+			if err := Start(cli.testnet, cli.infp); err != nil {
 				return err
 			}
 
diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go
index 08dab035f24..4a8a521bcdb 100644
--- a/test/e2e/runner/start.go
+++ b/test/e2e/runner/start.go
@@ -8,10 +8,10 @@ import (
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
-	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
 )
 
-func Start(testnet *e2e.Testnet) error {
+func Start(testnet *e2e.Testnet, p infra.Provider) error {
 	if len(testnet.Nodes) == 0 {
 		return fmt.Errorf("no nodes in testnet")
 	}
@@ -45,7 +45,12 @@ func Start(testnet *e2e.Testnet) error {
 	for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 {
 		node := nodeQueue[0]
 		nodeQueue = nodeQueue[1:]
-		if err := docker.ExecCompose(context.Background(), testnet.Dir, "up", "-d", node.Name); err != nil {
+		err := p.CreateNode(context.Background(), node)
+		if err != nil {
+			return err
+		}
+		err = p.StartTendermint(context.Background(), node)
+		if err != nil {
 			return err
 		}
 		if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
@@ -99,7 +104,12 @@ func Start(testnet *e2e.Testnet) error {
 
 		logger.Info("Starting catch up node", "node", node.Name, "height", node.StartAt)
 
-		if err := docker.ExecCompose(context.Background(), testnet.Dir, "up", "-d", node.Name); err != nil {
+		err := p.CreateNode(context.Background(), node)
+		if err != nil {
+			return err
+		}
+		err = p.StartTendermint(context.Background(), node)
+		if err != nil {
 			return err
 		}
 		status, err := waitForNode(node, node.StartAt, 3*time.Minute)

From af8d46e90848305a38be8ae8aa42b870ebd3e774 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:41:28 -0500
Subject: [PATCH 16/73] commit exec pkg

---
 test/e2e/pkg/exec/exec.go | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 test/e2e/pkg/exec/exec.go

diff --git a/test/e2e/pkg/exec/exec.go b/test/e2e/pkg/exec/exec.go
new file mode 100644
index 00000000000..9dcd793844a
--- /dev/null
+++ b/test/e2e/pkg/exec/exec.go
@@ -0,0 +1,34 @@
+package exec
+
+import (
+	"context"
+	"fmt"
+	"os"
+	osexec "os/exec"
+)
+
+// Command executes a shell command.
+func Command(ctx context.Context, args ...string) error {
+	// nolint: gosec
+	// G204: Subprocess launched with a potential tainted input or cmd arguments
+	cmd := osexec.CommandContext(ctx, args[0], args[1:]...)
+	out, err := cmd.CombinedOutput()
+	switch err := err.(type) {
+	case nil:
+		return nil
+	case *osexec.ExitError:
+		return fmt.Errorf("failed to run %q:\n%v", args, string(out))
+	default:
+		return err
+	}
+}
+
+// CommandVerbose executes a shell command while displaying its output.
+func CommandVerbose(ctx context.Context, args ...string) error {
+	// nolint: gosec
+	// G204: Subprocess launched with a potential tainted input or cmd arguments
+	cmd := osexec.CommandContext(ctx, args[0], args[1:]...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}

From 419d0d7295fa411851e9fdb9e86d859f0f7d7a0a Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:41:38 -0500
Subject: [PATCH 17/73] add ssh pkg

---
 test/e2e/pkg/ssh/ssh.go | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 test/e2e/pkg/ssh/ssh.go

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
new file mode 100644
index 00000000000..bd2619d3561
--- /dev/null
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -0,0 +1,33 @@
+package ssh
+
+import (
+	"os"
+	"path/filepath"
+
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+)
+
+func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
+	c, err := ssh.Dial("tcp", addr, cfg)
+	if err != nil {
+		return err
+	}
+	s, err := c.NewSession()
+	defer s.Close()
+	err = s.Run(cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func NewClientConfig(key string) (*ssh.ClientConfig, error) {
+	hkc, err := knownhosts.New(filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts"))
+	if err != nil {
+		return nil, err
+	}
+	return &ssh.ClientConfig{
+		HostKeyCallback: hkc,
+	}, nil
+}

From 31a37eed0211aae6c7db7f94375a0b2a4219fa6c Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:41:48 -0500
Subject: [PATCH 18/73] add digital ocean package

---
 .../pkg/infra/digitalocean/digitalocean.go    | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 test/e2e/pkg/infra/digitalocean/digitalocean.go

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
new file mode 100644
index 00000000000..3529a78b839
--- /dev/null
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -0,0 +1,44 @@
+package digitalocean
+
+import (
+	"context"
+	"fmt"
+
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
+	e2essh "github.com/tendermint/tendermint/test/e2e/pkg/ssh"
+	"golang.org/x/crypto/ssh"
+)
+
+const (
+	sshPort     = 22
+	testappName = "testappd"
+)
+
+var _ infra.Provider = &Provider{}
+
+// Provider implements a docker-compose backed infrastructure provider.
+type Provider struct {
+	Testnet            *e2e.Testnet
+	InfrastructureData e2e.InfrastructureData
+	SSHConfig          *ssh.ClientConfig
+}
+
+// Noop currently. Setup is performed externally to the e2e test tool.
+func (p *Provider) Setup() error {
+	return nil
+}
+
+// Noop currently. Node creation is currently performed externally to the e2e test tool.
+func (p Provider) CreateNode(ctx context.Context, n *e2e.Node) error {
+	return nil
+}
+func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort), fmt.Sprintf("systemctl start %s", testappName))
+}
+func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort), fmt.Sprintf("systemctl -s SIGTERM %s", testappName))
+}
+func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort), fmt.Sprintf("systemctl -s SIGKILL %s", testappName))
+}

From fcb92561ce44b40c7ac19e3ba5d582b66d1f4b0e Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:46:00 -0500
Subject: [PATCH 19/73] add client creation logic

---
 test/e2e/pkg/ssh/ssh.go | 14 +++++++++++++-
 test/e2e/runner/main.go | 21 +++++++++++++++++++--
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index bd2619d3561..f672f0cae4d 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -22,12 +22,24 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 	return nil
 }
 
-func NewClientConfig(key string) (*ssh.ClientConfig, error) {
+func NewClientConfig(keyFile string) (*ssh.ClientConfig, error) {
 	hkc, err := knownhosts.New(filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts"))
 	if err != nil {
 		return nil, err
 	}
+	key, err := os.ReadFile(keyFile)
+	if err != nil {
+		return nil, err
+	}
+	signer, err := ssh.ParsePrivateKey(key)
+	if err != nil {
+		return nil, err
+	}
 	return &ssh.ClientConfig{
 		HostKeyCallback: hkc,
+		Auth: []ssh.AuthMethod{
+			ssh.PublicKeys(signer),
+		},
+		HostKeyAlgorithms: []string{ssh.KeyAlgoED25519},
 	}, nil
 }
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index f7541cc7386..291deb2eb37 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"path/filepath"
 	"strconv"
 
 	"github.com/spf13/cobra"
@@ -13,7 +14,9 @@ import (
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra/digitalocean"
 	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
+	e2essh "github.com/tendermint/tendermint/test/e2e/pkg/ssh"
 )
 
 const randomSeed = 2308084734268
@@ -85,9 +88,23 @@ func NewCLI() *CLI {
 			}
 
 			cli.testnet = testnet
-			cli.infp = &infra.NoopProvider{}
-			if inft == "docker" {
+			switch inft {
+			case "docker":
 				cli.infp = &docker.Provider{Testnet: testnet}
+			case "digitalocean":
+
+				// TMP: hardcoded key
+				cfg, err := e2essh.NewClientConfig(filepath.Join(os.Getenv("HOME"), ".ssh", "PVT_KEY"))
+				if err != nil {
+					return err
+				}
+				cli.infp = &digitalocean.Provider{
+					Testnet:            testnet,
+					InfrastructureData: ifd,
+					SSHConfig:          cfg,
+				}
+			default:
+				cli.infp = &infra.NoopProvider{}
 			}
 			return nil
 		},

From 5e8028da5d736ea293fae70600ac3b1eaa7611ad Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 14:12:12 -0500
Subject: [PATCH 20/73] use ssh agent to auth to remote

---
 test/e2e/pkg/infrastructure.go |  4 ++++
 test/e2e/pkg/ssh/ssh.go        | 18 +++++++++---------
 test/e2e/pkg/testnet.go        |  8 ++++----
 test/e2e/runner/main.go        | 14 +++++++++-----
 test/e2e/runner/start.go       |  6 +++---
 5 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/test/e2e/pkg/infrastructure.go b/test/e2e/pkg/infrastructure.go
index 2fc0e4bac6e..582ed491aec 100644
--- a/test/e2e/pkg/infrastructure.go
+++ b/test/e2e/pkg/infrastructure.go
@@ -38,6 +38,7 @@ type InfrastructureData struct {
 // one of the nodes in the testnet.
 type InstanceData struct {
 	IPAddress net.IP `json:"ip_address"`
+	Port      uint32 `json:"port"`
 }
 
 func NewDockerInfrastructureData(m Manifest) (InfrastructureData, error) {
@@ -49,6 +50,8 @@ func NewDockerInfrastructureData(m Manifest) (InfrastructureData, error) {
 	if err != nil {
 		return InfrastructureData{}, fmt.Errorf("invalid IP network address %q: %w", netAddress, err)
 	}
+
+	portGen := newPortGenerator(proxyPortFirst)
 	ipGen := newIPGenerator(ipNet)
 	ifd := InfrastructureData{
 		Provider:  "docker",
@@ -58,6 +61,7 @@ func NewDockerInfrastructureData(m Manifest) (InfrastructureData, error) {
 	for name := range m.Nodes {
 		ifd.Instances[name] = InstanceData{
 			IPAddress: ipGen.Next(),
+			Port:      portGen.Next(),
 		}
 	}
 	return ifd, nil
diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index f672f0cae4d..ea74e23d526 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 
 	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/agent"
 	"golang.org/x/crypto/ssh/knownhosts"
 )
 
@@ -22,24 +23,23 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 	return nil
 }
 
-func NewClientConfig(keyFile string) (*ssh.ClientConfig, error) {
+func NewClientConfig(ac agent.ExtendedAgent) (*ssh.ClientConfig, error) {
 	hkc, err := knownhosts.New(filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts"))
 	if err != nil {
 		return nil, err
 	}
-	key, err := os.ReadFile(keyFile)
+	signers, err := ac.Signers()
 	if err != nil {
 		return nil, err
 	}
-	signer, err := ssh.ParsePrivateKey(key)
-	if err != nil {
-		return nil, err
+	am := make([]ssh.AuthMethod, 0, len(signers))
+	for _, signer := range signers {
+		am = append(am, ssh.PublicKeys(signer))
 	}
 	return &ssh.ClientConfig{
-		HostKeyCallback: hkc,
-		Auth: []ssh.AuthMethod{
-			ssh.PublicKeys(signer),
-		},
+		User:              "root",
+		HostKeyCallback:   hkc,
+		Auth:              am,
 		HostKeyAlgorithms: []string{ssh.KeyAlgoED25519},
 	}, nil
 }
diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go
index 030d542f7dc..65fa53119c9 100644
--- a/test/e2e/pkg/testnet.go
+++ b/test/e2e/pkg/testnet.go
@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -102,7 +103,6 @@ type Node struct {
 func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Testnet, error) {
 	dir := strings.TrimSuffix(fname, filepath.Ext(fname))
 	keyGen := newKeyGenerator(randomSeed)
-	proxyPortGen := newPortGenerator(proxyPortFirst)
 	_, ipNet, err := net.ParseCIDR(ifd.Network)
 	if err != nil {
 		return nil, fmt.Errorf("invalid IP network address %q: %w", ifd.Network, err)
@@ -153,7 +153,7 @@ func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Test
 			PrivvalKey:       keyGen.Generate(manifest.KeyType),
 			NodeKey:          keyGen.Generate("ed25519"),
 			IP:               ind.IPAddress,
-			ProxyPort:        proxyPortGen.Next(),
+			ProxyPort:        ind.Port,
 			Mode:             ModeValidator,
 			SyncApp:          nodeManifest.SyncApp,
 			Database:         "goleveldb",
@@ -297,7 +297,7 @@ func (n Node) Validate(testnet Testnet) error {
 			return fmt.Errorf("local port %v must be >1024", n.ProxyPort)
 		}
 		for _, peer := range testnet.Nodes {
-			if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort {
+			if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort && bytes.Equal(peer.IP, n.IP) {
 				return fmt.Errorf("peer %q also has local port %v", peer.Name, n.ProxyPort)
 			}
 		}
@@ -437,7 +437,7 @@ func (n Node) AddressRPC() string {
 
 // Client returns an RPC client for a node.
 func (n Node) Client() (*rpchttp.HTTP, error) {
-	return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort), "/websocket")
+	return rpchttp.New(fmt.Sprintf("http://%s:%v", n.IP, n.ProxyPort), "/websocket")
 }
 
 // Stateless returns true if the node is either a seed node or a light node
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index 291deb2eb37..989ab296e03 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -5,11 +5,12 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
+	"net"
 	"os"
-	"path/filepath"
 	"strconv"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/crypto/ssh/agent"
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
@@ -91,10 +92,13 @@ func NewCLI() *CLI {
 			switch inft {
 			case "docker":
 				cli.infp = &docker.Provider{Testnet: testnet}
-			case "digitalocean":
-
-				// TMP: hardcoded key
-				cfg, err := e2essh.NewClientConfig(filepath.Join(os.Getenv("HOME"), ".ssh", "PVT_KEY"))
+			case "digital-ocean":
+				c, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")) // TODO(williambanfield): Ensure this is a consistent location for the authentication socket of the agent.
+				if err != nil {
+					return err
+				}
+				ac := agent.NewClient(c)
+				cfg, err := e2essh.NewClientConfig(ac)
 				if err != nil {
 					return err
 				}
diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go
index 4a8a521bcdb..72ee3db6d41 100644
--- a/test/e2e/runner/start.go
+++ b/test/e2e/runner/start.go
@@ -56,7 +56,7 @@ func Start(testnet *e2e.Testnet, p infra.Provider) error {
 		if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
 			return err
 		}
-		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort))
+		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://%s:%v", node.Name, node.IP, node.ProxyPort))
 	}
 
 	networkHeight := testnet.InitialHeight
@@ -116,8 +116,8 @@ func Start(testnet *e2e.Testnet, p infra.Provider) error {
 		if err != nil {
 			return err
 		}
-		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://127.0.0.1:%v at height %v",
-			node.Name, node.ProxyPort, status.SyncInfo.LatestBlockHeight))
+		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://%s:%v at height %v",
+			node.Name, node.IP, node.ProxyPort, status.SyncInfo.LatestBlockHeight))
 	}
 
 	return nil

From 46ab7a376d6826f3204990ee21a42ba3ba70fa46 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 17:48:54 -0500
Subject: [PATCH 21/73] plump infra data into tests

---
 test/e2e/pkg/infrastructure.go |  6 ++++++
 test/e2e/runner/main.go        |  6 ++++--
 test/e2e/runner/test.go        | 12 +++++++++++-
 test/e2e/tests/e2e_test.go     | 21 +++++++++++++++++++--
 4 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/test/e2e/pkg/infrastructure.go b/test/e2e/pkg/infrastructure.go
index 582ed491aec..041e1a8d0c4 100644
--- a/test/e2e/pkg/infrastructure.go
+++ b/test/e2e/pkg/infrastructure.go
@@ -17,6 +17,7 @@ const (
 // InfrastructureData contains the relevant information for a set of existing
 // infrastructure that is to be used for running a testnet.
 type InfrastructureData struct {
+	path string
 
 	// Provider is the name of infrastructure provider backing the testnet.
 	// For example, 'docker' if it is running locally in a docker network or
@@ -41,6 +42,10 @@ type InstanceData struct {
 	Port      uint32 `json:"port"`
 }
 
+func (i InfrastructureData) Path() string {
+	return i.path
+}
+
 func NewDockerInfrastructureData(m Manifest) (InfrastructureData, error) {
 	netAddress := dockerIPv4CIDR
 	if m.IPv6 {
@@ -80,5 +85,6 @@ func InfrastructureDataFromFile(p string) (InfrastructureData, error) {
 	if ifd.Network == "" {
 		ifd.Network = globalIPv4CIDR
 	}
+	ifd.path = p
 	return ifd, nil
 }
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index 989ab296e03..622c6e0579b 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -34,6 +34,7 @@ type CLI struct {
 	testnet  *e2e.Testnet
 	preserve bool
 	infp     infra.Provider
+	ifd      e2e.InfrastructureData
 }
 
 // NewCLI sets up the CLI.
@@ -82,6 +83,7 @@ func NewCLI() *CLI {
 			default:
 				return fmt.Errorf("unknown infrastructure type '%s'", inft)
 			}
+			cli.ifd = ifd
 
 			testnet, err := e2e.LoadTestnet(m, file, ifd)
 			if err != nil {
@@ -166,7 +168,7 @@ func NewCLI() *CLI {
 			if err := Wait(cli.testnet, 5); err != nil { // wait for network to settle before tests
 				return err
 			}
-			if err := Test(cli.testnet); err != nil {
+			if err := Test(cli.testnet, cli.ifd); err != nil {
 				return err
 			}
 			if !cli.preserve {
@@ -281,7 +283,7 @@ func NewCLI() *CLI {
 		Use:   "test",
 		Short: "Runs test cases against a running testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return Test(cli.testnet)
+			return Test(cli.testnet, cli.ifd)
 		},
 	})
 
diff --git a/test/e2e/runner/test.go b/test/e2e/runner/test.go
index 4671a81d262..819d5b9ac08 100644
--- a/test/e2e/runner/test.go
+++ b/test/e2e/runner/test.go
@@ -9,13 +9,23 @@ import (
 )
 
 // Test runs test cases under tests/
-func Test(testnet *e2e.Testnet) error {
+func Test(testnet *e2e.Testnet, ifd e2e.InfrastructureData) error {
 	logger.Info("Running tests in ./tests/...")
 
 	err := os.Setenv("E2E_MANIFEST", testnet.File)
 	if err != nil {
 		return err
 	}
+	if p := ifd.Path(); p != "" {
+		err = os.Setenv("INFRASTRUCTURE_DATA", p)
+		if err != nil {
+			return err
+		}
+	}
+	err = os.Setenv("INFRASTRUCTURE_TYPE", ifd.Provider)
+	if err != nil {
+		return err
+	}
 
 	return exec.CommandVerbose(context.Background(), "go", "test", "-count", "1", "./tests/...")
 }
diff --git a/test/e2e/tests/e2e_test.go b/test/e2e/tests/e2e_test.go
index 5df3309d418..8e5ee9379d7 100644
--- a/test/e2e/tests/e2e_test.go
+++ b/test/e2e/tests/e2e_test.go
@@ -73,7 +73,14 @@ func loadTestnet(t *testing.T) e2e.Testnet {
 	if !filepath.IsAbs(manifestFile) {
 		manifestFile = filepath.Join("..", manifestFile)
 	}
-
+	ifdType := os.Getenv("INFRASTRUCTURE_DATA")
+	ifdFile := os.Getenv("INFRASTRUCTURE_FILE")
+	if ifdFile == "" && ifdType != "docker" {
+		t.Skip("INFRASTRUCTURE_DATA not set and INFRASTRUCTURE_TYPE is not docker")
+	}
+	if !filepath.IsAbs(ifdFile) {
+		manifestFile = filepath.Join("..", manifestFile)
+	}
 	testnetCacheMtx.Lock()
 	defer testnetCacheMtx.Unlock()
 	if testnet, ok := testnetCache[manifestFile]; ok {
@@ -81,7 +88,17 @@ func loadTestnet(t *testing.T) e2e.Testnet {
 	}
 	m, err := e2e.LoadManifest(manifestFile)
 	require.NoError(t, err)
-	ifd, err := e2e.NewDockerInfrastructureData(m)
+
+	var ifd e2e.InfrastructureData
+	switch ifdType {
+	case "docker":
+		ifd, err = e2e.NewDockerInfrastructureData(m)
+		require.NoError(t, err)
+	case "digital-ocean":
+		ifd, err = e2e.InfrastructureDataFromFile(manifestFile)
+		require.NoError(t, err)
+	default:
+	}
 	require.NoError(t, err)
 
 	testnet, err := e2e.LoadTestnet(m, manifestFile, ifd)

From 0197fec56c8a45837fe936920f71431cdfd5ec82 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 18:02:16 -0500
Subject: [PATCH 22/73] move agent socket dial to client connection creation

---
 test/e2e/pkg/ssh/ssh.go | 8 +++++++-
 test/e2e/runner/main.go | 9 +--------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index ea74e23d526..07be10ccc05 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -1,6 +1,7 @@
 package ssh
 
 import (
+	"net"
 	"os"
 	"path/filepath"
 
@@ -23,7 +24,12 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 	return nil
 }
 
-func NewClientConfig(ac agent.ExtendedAgent) (*ssh.ClientConfig, error) {
+func NewClientConfig() (*ssh.ClientConfig, error) {
+	c, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
+	if err != nil {
+		return nil, err
+	}
+	ac := agent.NewClient(c)
 	hkc, err := knownhosts.New(filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts"))
 	if err != nil {
 		return nil, err
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index 622c6e0579b..be11e05999e 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -5,12 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
-	"net"
 	"os"
 	"strconv"
 
 	"github.com/spf13/cobra"
-	"golang.org/x/crypto/ssh/agent"
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
@@ -95,12 +93,7 @@ func NewCLI() *CLI {
 			case "docker":
 				cli.infp = &docker.Provider{Testnet: testnet}
 			case "digital-ocean":
-				c, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")) // TODO(williambanfield): Ensure this is a consistent location for the authentication socket of the agent.
-				if err != nil {
-					return err
-				}
-				ac := agent.NewClient(c)
-				cfg, err := e2essh.NewClientConfig(ac)
+				cfg, err := e2essh.NewClientConfig()
 				if err != nil {
 					return err
 				}

From b1c65de1c2cb6951b066d6c5dd5470316236cca1 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 18:05:10 -0500
Subject: [PATCH 23/73] error if auth sock not defined

---
 test/e2e/pkg/ssh/ssh.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index 07be10ccc05..08a5324e65b 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -1,6 +1,7 @@
 package ssh
 
 import (
+	"errors"
 	"net"
 	"os"
 	"path/filepath"
@@ -25,7 +26,11 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 }
 
 func NewClientConfig() (*ssh.ClientConfig, error) {
-	c, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
+	ss := os.Getenv("SSH_AUTH_SOCK")
+	if ss == "" {
+		return nil, errors.New("SSH_AUTH_SOCK environment variable is empty. Is the ssh-agent running?")
+	}
+	c, err := net.Dial("unix", ss)
 	if err != nil {
 		return nil, err
 	}

From 01c896d6221139442357967d44fedb7a986424f6 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 18:37:35 -0500
Subject: [PATCH 24/73] allow unknown hosts to be accessed

---
 test/e2e/pkg/ssh/ssh.go | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index 08a5324e65b..5246ada838f 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -2,6 +2,8 @@ package ssh
 
 import (
 	"errors"
+	"fmt"
+	"log"
 	"net"
 	"os"
 	"path/filepath"
@@ -49,8 +51,28 @@ func NewClientConfig() (*ssh.ClientConfig, error) {
 	}
 	return &ssh.ClientConfig{
 		User:              "root",
-		HostKeyCallback:   hkc,
+		HostKeyCallback:   hkcWrapper(hkc),
 		Auth:              am,
 		HostKeyAlgorithms: []string{ssh.KeyAlgoED25519},
 	}, nil
 }
+
+func hkcWrapper(hkc ssh.HostKeyCallback) ssh.HostKeyCallback {
+	return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+		err := hkc(hostname, remote, key)
+		if err == nil {
+			return nil
+		}
+		ke := &knownhosts.KeyError{}
+		if errors.As(err, &ke) && len(ke.Want) == 0 {
+			h, _, err := net.SplitHostPort(hostname)
+			if err != nil {
+				panic(fmt.Errorf("hostname incorrectly formatted: %w", err))
+			}
+			log.Printf("ignoring knownhosts error for unknown host: %s", h)
+			return nil
+		}
+
+		return err
+	}
+}

From d2ef43b449babba57ce8582dcf89106d1eb9f22f Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 19:00:25 -0500
Subject: [PATCH 25/73] first attempt at disconnect

---
 test/e2e/pkg/infra/digitalocean/digitalocean.go | 14 ++++++++++++++
 test/e2e/pkg/infra/docker/docker.go             |  6 ++++++
 test/e2e/pkg/infra/provider.go                  |  4 ++++
 test/e2e/pkg/ssh/ssh.go                         | 16 ++++++++++++++++
 test/e2e/runner/main.go                         |  4 ++--
 test/e2e/runner/perturb.go                      | 11 ++++++-----
 6 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index 3529a78b839..40823bd8538 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -42,3 +42,17 @@ func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
 func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
 	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort), fmt.Sprintf("systemctl -s SIGKILL %s", testappName))
 }
+func (p Provider) Disconnect(ctx context.Context, n *e2e.Node) error {
+	return e2essh.MultiExec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort),
+		"iptables -A INPUT -p tcp --destination-port 26656 -j DROP",
+		"iptables -A OUTPUT -p tcp --destination-port 26656 -j DROP",
+		"service iptables save",
+	)
+}
+func (p Provider) Connect(ctx context.Context, n *e2e.Node) error {
+	return e2essh.MultiExec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort),
+		"iptables -D INPUT -p tcp --destination-port 26656 -j DROP",
+		"iptables -D OUTPUT -p tcp --destination-port 26656 -j DROP",
+		"service iptables save",
+	)
+}
diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index a177002985b..bc33f8ab59e 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -47,6 +47,12 @@ func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
 func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
 	return ExecCompose(ctx, p.Testnet.Dir, "kill", "-s", "SIGKILL", n.Name)
 }
+func (p Provider) Connect(ctx context.Context, n *e2e.Node) error {
+	return Exec(ctx, "network", "connect", p.Testnet.Name+"_"+p.Testnet.Name, n.Name)
+}
+func (p Provider) Disconnect(ctx context.Context, n *e2e.Node) error {
+	return Exec(ctx, "network", "disconnect", p.Testnet.Name+"_"+p.Testnet.Name, n.Name)
+}
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index 0fdcc86baa8..4de74b9bd66 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -18,6 +18,8 @@ type Provider interface {
 	StartTendermint(context.Context, *e2e.Node) error
 	KillTendermint(context.Context, *e2e.Node) error
 	TerminateTendermint(context.Context, *e2e.Node) error
+	Connect(context.Context, *e2e.Node) error
+	Disconnect(context.Context, *e2e.Node) error
 }
 
 // NoopProvider implements the provider interface by performing noops for every
@@ -31,5 +33,7 @@ func (NoopProvider) CreateNode(_ context.Context, _ *e2e.Node) error          {
 func (NoopProvider) StartTendermint(_ context.Context, _ *e2e.Node) error     { return nil }
 func (NoopProvider) TerminateTendermint(_ context.Context, _ *e2e.Node) error { return nil }
 func (NoopProvider) KillTendermint(_ context.Context, _ *e2e.Node) error      { return nil }
+func (NoopProvider) Connect(_ context.Context, _ *e2e.Node) error             { return nil }
+func (NoopProvider) Disconnect(_ context.Context, _ *e2e.Node) error          { return nil }
 
 var _ Provider = NoopProvider{}
diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index 5246ada838f..cdca8d08765 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -27,6 +27,22 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 	return nil
 }
 
+func MultiExec(cfg *ssh.ClientConfig, addr string, cmds ...string) error {
+	c, err := ssh.Dial("tcp", addr, cfg)
+	s, err := c.NewSession()
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+	for _, cmd := range cmds {
+		err := s.Run(cmd)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func NewClientConfig() (*ssh.ClientConfig, error) {
 	ss := os.Getenv("SSH_AUTH_SOCK")
 	if ss == "" {
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index be11e05999e..3b7a6f3f857 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -137,7 +137,7 @@ func NewCLI() *CLI {
 			}
 
 			if cli.testnet.HasPerturbations() {
-				if err := Perturb(cli.testnet); err != nil {
+				if err := Perturb(cli.testnet, cli.infp); err != nil {
 					return err
 				}
 				if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
@@ -210,7 +210,7 @@ func NewCLI() *CLI {
 		Use:   "perturb",
 		Short: "Perturbs the Docker testnet, e.g. by restarting or disconnecting nodes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return Perturb(cli.testnet)
+			return Perturb(cli.testnet, cli.infp)
 		},
 	})
 
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index dfd0ff8b72b..9d90f59aac7 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -8,14 +8,15 @@ import (
 	"github.com/tendermint/tendermint/libs/log"
 	rpctypes "github.com/tendermint/tendermint/rpc/core/types"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
 	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
 )
 
 // Perturbs a running testnet.
-func Perturb(testnet *e2e.Testnet) error {
+func Perturb(testnet *e2e.Testnet, ifp infra.Provider) error {
 	for _, node := range testnet.Nodes {
 		for _, perturbation := range node.Perturbations {
-			_, err := PerturbNode(node, perturbation)
+			_, err := PerturbNode(node, perturbation, ifp)
 			if err != nil {
 				return err
 			}
@@ -27,16 +28,16 @@ func Perturb(testnet *e2e.Testnet) error {
 
 // PerturbNode perturbs a node with a given perturbation, returning its status
 // after recovering.
-func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
+func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation, ifp infra.Provider) (*rpctypes.ResultStatus, error) {
 	testnet := node.Testnet
 	switch perturbation {
 	case e2e.PerturbationDisconnect:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Disconnecting node %v...", node.Name))
-		if err := docker.Exec(context.Background(), "network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name) ; err != nil {
+		if err := ifp.Disconnect(context.Background(), node); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := docker.Exec(context.Background(), "network", "connect", testnet.Name+"_"+testnet.Name, node.Name) ; err != nil {
+		if err := ifp.Connect(context.Background(), node); err != nil {
 			return nil, err
 		}
 

From f9799ca469f405b836247439dfbf4e54825cf55a Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 20:15:10 -0500
Subject: [PATCH 26/73] reject with reset

---
 test/e2e/pkg/infra/digitalocean/digitalocean.go | 10 ++++------
 test/e2e/pkg/ssh/ssh.go                         | 15 ++++++++++++---
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index 40823bd8538..cdc18d28ef8 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -44,15 +44,13 @@ func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
 }
 func (p Provider) Disconnect(ctx context.Context, n *e2e.Node) error {
 	return e2essh.MultiExec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort),
-		"iptables -A INPUT -p tcp --destination-port 26656 -j DROP",
-		"iptables -A OUTPUT -p tcp --destination-port 26656 -j DROP",
-		"service iptables save",
+		"iptables -A INPUT -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset",
+		"iptables -A OUTPUT -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset",
 	)
 }
 func (p Provider) Connect(ctx context.Context, n *e2e.Node) error {
 	return e2essh.MultiExec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort),
-		"iptables -D INPUT -p tcp --destination-port 26656 -j DROP",
-		"iptables -D OUTPUT -p tcp --destination-port 26656 -j DROP",
-		"service iptables save",
+		"iptables -D INPUT -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset",
+		"iptables -D OUTPUT -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset",
 	)
 }
diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index cdca8d08765..bd3924627d8 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -18,7 +18,11 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 	if err != nil {
 		return err
 	}
+	defer c.Close()
 	s, err := c.NewSession()
+	if err != nil {
+		return err
+	}
 	defer s.Close()
 	err = s.Run(cmd)
 	if err != nil {
@@ -29,16 +33,21 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 
 func MultiExec(cfg *ssh.ClientConfig, addr string, cmds ...string) error {
 	c, err := ssh.Dial("tcp", addr, cfg)
-	s, err := c.NewSession()
 	if err != nil {
 		return err
 	}
-	defer s.Close()
+	defer c.Close()
 	for _, cmd := range cmds {
-		err := s.Run(cmd)
+		s, err := c.NewSession()
+		if err != nil {
+			return err
+		}
+		err = s.Run(cmd)
 		if err != nil {
+			s.Close()
 			return err
 		}
+		s.Close()
 	}
 	return nil
 }

From e49f6aecdbc184c83a2e91849bfb9f2f432255f0 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:15:42 -0500
Subject: [PATCH 27/73] shim in start and stop methods

---
 test/e2e/pkg/infra/docker/docker.go |  3 +++
 test/e2e/pkg/infra/provider.go      | 13 ++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 04811aab5e1..4e1c1792e6c 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -2,6 +2,7 @@ package docker
 
 import (
 	"bytes"
+	"context"
 	"os"
 	"path/filepath"
 	"text/template"
@@ -32,6 +33,8 @@ func (p *Provider) Setup() error {
 	}
 	return nil
 }
+func (Provider) StartNode(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
+func (Provider) StopNode(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index 03b821de384..34cbb8be0dc 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -1,5 +1,11 @@
 package infra
 
+import (
+	"context"
+
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+)
+
 // Provider defines an API for manipulating the infrastructure of a
 // specific set of testnet infrastructure.
 type Provider interface {
@@ -7,6 +13,9 @@ type Provider interface {
 	// Setup generates any necessary configuration for the infrastructure
 	// provider during testnet setup.
 	Setup() error
+
+	StartNode(context.Context, *e2e.Node) error
+	StopNode(context.Context, *e2e.Node) error
 }
 
 // NoopProvider implements the provider interface by performing noops for every
@@ -15,6 +24,8 @@ type Provider interface {
 type NoopProvider struct {
 }
 
-func (NoopProvider) Setup() error { return nil }
+func (NoopProvider) Setup() error                                   { return nil }
+func (NoopProvider) StartNode(_ context.Context, _ *e2e.Node) error { return nil }
+func (NoopProvider) StopNode(_ context.Context, _ *e2e.Node) error  { return nil }
 
 var _ Provider = NoopProvider{}

From 1a831b2fcbece3d6cbd7121087f24c206a4757b5 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:17:22 -0500
Subject: [PATCH 28/73] rename to 'tendermint'

---
 test/e2e/pkg/infra/docker/docker.go |  4 ++--
 test/e2e/pkg/infra/provider.go      | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 4e1c1792e6c..979b00b5b1d 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -33,8 +33,8 @@ func (p *Provider) Setup() error {
 	}
 	return nil
 }
-func (Provider) StartNode(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
-func (Provider) StopNode(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
+func (Provider) StartTendermint(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
+func (Provider) StopTendermint(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index 34cbb8be0dc..47fa7a702d6 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -14,8 +14,8 @@ type Provider interface {
 	// provider during testnet setup.
 	Setup() error
 
-	StartNode(context.Context, *e2e.Node) error
-	StopNode(context.Context, *e2e.Node) error
+	StartTendermint(context.Context, *e2e.Node) error
+	StopTendermint(context.Context, *e2e.Node) error
 }
 
 // NoopProvider implements the provider interface by performing noops for every
@@ -24,8 +24,8 @@ type Provider interface {
 type NoopProvider struct {
 }
 
-func (NoopProvider) Setup() error                                   { return nil }
-func (NoopProvider) StartNode(_ context.Context, _ *e2e.Node) error { return nil }
-func (NoopProvider) StopNode(_ context.Context, _ *e2e.Node) error  { return nil }
+func (NoopProvider) Setup() error                                         { return nil }
+func (NoopProvider) StartTendermint(_ context.Context, _ *e2e.Node) error { return nil }
+func (NoopProvider) StopTendermint(_ context.Context, _ *e2e.Node) error  { return nil }
 
 var _ Provider = NoopProvider{}

From 64b03402513d377b3e7f74a07a2b484508bb6c87 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:19:15 -0500
Subject: [PATCH 29/73] rename to 'kill tendermint'

---
 test/e2e/pkg/infra/docker/docker.go | 2 +-
 test/e2e/pkg/infra/provider.go      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 979b00b5b1d..a6ccf89783b 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -34,7 +34,7 @@ func (p *Provider) Setup() error {
 	return nil
 }
 func (Provider) StartTendermint(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
-func (Provider) StopTendermint(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
+func (Provider) KillTendermint(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index 47fa7a702d6..1822d895ede 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -15,7 +15,7 @@ type Provider interface {
 	Setup() error
 
 	StartTendermint(context.Context, *e2e.Node) error
-	StopTendermint(context.Context, *e2e.Node) error
+	KillTendermint(context.Context, *e2e.Node) error
 }
 
 // NoopProvider implements the provider interface by performing noops for every
@@ -26,6 +26,6 @@ type NoopProvider struct {
 
 func (NoopProvider) Setup() error                                         { return nil }
 func (NoopProvider) StartTendermint(_ context.Context, _ *e2e.Node) error { return nil }
-func (NoopProvider) StopTendermint(_ context.Context, _ *e2e.Node) error  { return nil }
+func (NoopProvider) KillTendermint(_ context.Context, _ *e2e.Node) error  { return nil }
 
 var _ Provider = NoopProvider{}

From 23cb6ad834f6fbe89857713d3f159435498b100b Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:21:26 -0500
Subject: [PATCH 30/73] add terminate vs kill

---
 test/e2e/pkg/infra/docker/docker.go | 5 +++--
 test/e2e/pkg/infra/provider.go      | 8 +++++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index a6ccf89783b..b72dcfa764a 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -33,8 +33,9 @@ func (p *Provider) Setup() error {
 	}
 	return nil
 }
-func (Provider) StartTendermint(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
-func (Provider) KillTendermint(_ context.Context, _ *e2e.Node) error  { panic("unimplemented") }
+func (Provider) StartTendermint(_ context.Context, _ *e2e.Node) error     { panic("unimplemented") }
+func (Provider) TerminateTendermint(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
+func (Provider) KillTendermint(_ context.Context, _ *e2e.Node) error      { panic("unimplemented") }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index 1822d895ede..bff50eb6d20 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -16,6 +16,7 @@ type Provider interface {
 
 	StartTendermint(context.Context, *e2e.Node) error
 	KillTendermint(context.Context, *e2e.Node) error
+	TerminateTendermint(context.Context, *e2e.Node) error
 }
 
 // NoopProvider implements the provider interface by performing noops for every
@@ -24,8 +25,9 @@ type Provider interface {
 type NoopProvider struct {
 }
 
-func (NoopProvider) Setup() error                                         { return nil }
-func (NoopProvider) StartTendermint(_ context.Context, _ *e2e.Node) error { return nil }
-func (NoopProvider) KillTendermint(_ context.Context, _ *e2e.Node) error  { return nil }
+func (NoopProvider) Setup() error                                             { return nil }
+func (NoopProvider) StartTendermint(_ context.Context, _ *e2e.Node) error     { return nil }
+func (NoopProvider) TerminateTendermint(_ context.Context, _ *e2e.Node) error { return nil }
+func (NoopProvider) KillTendermint(_ context.Context, _ *e2e.Node) error      { return nil }
 
 var _ Provider = NoopProvider{}

From 6117ad38aa9a6ae2017e2e11981bf19c2cb71edd Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:34:56 -0500
Subject: [PATCH 31/73] add command package

---
 test/e2e/runner/cleanup.go |  6 ++++--
 test/e2e/runner/exec.go    | 34 ++++++----------------------------
 test/e2e/runner/test.go    |  4 +++-
 3 files changed, 13 insertions(+), 31 deletions(-)

diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go
index fbe7aa0b4cc..74e84f7fc72 100644
--- a/test/e2e/runner/cleanup.go
+++ b/test/e2e/runner/cleanup.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -8,6 +9,7 @@ import (
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
 )
 
 // Cleanup removes the Docker Compose containers and testnet directory.
@@ -32,13 +34,13 @@ func cleanupDocker() error {
 	// does this by default. Ugly, but works.
 	xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)`
 
-	err := exec("bash", "-c", fmt.Sprintf(
+	err := exec.Command(context.Background(), "bash", "-c", fmt.Sprintf(
 		"docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR))
 	if err != nil {
 		return err
 	}
 
-	err = exec("bash", "-c", fmt.Sprintf(
+	err = exec.Command(context.Background(), "bash", "-c", fmt.Sprintf(
 		"docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR))
 	if err != nil {
 		return err
diff --git a/test/e2e/runner/exec.go b/test/e2e/runner/exec.go
index e6e47ca0a96..eedc447336c 100644
--- a/test/e2e/runner/exec.go
+++ b/test/e2e/runner/exec.go
@@ -1,49 +1,27 @@
 package main
 
 import (
-	"fmt"
-	"os"
-	osexec "os/exec"
 	"path/filepath"
-)
-
-// execute executes a shell command.
-func exec(args ...string) error {
-	cmd := osexec.Command(args[0], args[1:]...) //nolint:gosec
-	out, err := cmd.CombinedOutput()
-	switch err := err.(type) {
-	case nil:
-		return nil
-	case *osexec.ExitError:
-		return fmt.Errorf("failed to run %q:\n%v", args, string(out))
-	default:
-		return err
-	}
-}
 
-// execVerbose executes a shell command while displaying its output.
-func execVerbose(args ...string) error {
-	cmd := osexec.Command(args[0], args[1:]...) //nolint:gosec
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
-}
+	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
+	"golang.org/x/net/context"
+)
 
 // execCompose runs a Docker Compose command for a testnet.
 func execCompose(dir string, args ...string) error {
-	return exec(append(
+	return exec.Command(context.Background(), append(
 		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
 		args...)...)
 }
 
 // execComposeVerbose runs a Docker Compose command for a testnet and displays its output.
 func execComposeVerbose(dir string, args ...string) error {
-	return execVerbose(append(
+	return exec.CommandVerbose(context.Background(), append(
 		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
 		args...)...)
 }
 
 // execDocker runs a Docker command.
 func execDocker(args ...string) error {
-	return exec(append([]string{"docker"}, args...)...)
+	return exec.Command(context.Background(), append([]string{"docker"}, args...)...)
 }
diff --git a/test/e2e/runner/test.go b/test/e2e/runner/test.go
index 834ce6f2d04..4671a81d262 100644
--- a/test/e2e/runner/test.go
+++ b/test/e2e/runner/test.go
@@ -1,9 +1,11 @@
 package main
 
 import (
+	"context"
 	"os"
 
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
 )
 
 // Test runs test cases under tests/
@@ -15,5 +17,5 @@ func Test(testnet *e2e.Testnet) error {
 		return err
 	}
 
-	return execVerbose("go", "test", "-count", "1", "./tests/...")
+	return exec.CommandVerbose(context.Background(), "go", "test", "-count", "1", "./tests/...")
 }

From 11ca11a8cc3fcd696a4db3b3337b53d2b31f8c74 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:38:26 -0500
Subject: [PATCH 32/73] add param names

---
 test/e2e/pkg/infra/docker/docker.go | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index b72dcfa764a..7ba8ca3cd4d 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -33,9 +33,15 @@ func (p *Provider) Setup() error {
 	}
 	return nil
 }
-func (Provider) StartTendermint(_ context.Context, _ *e2e.Node) error     { panic("unimplemented") }
-func (Provider) TerminateTendermint(_ context.Context, _ *e2e.Node) error { panic("unimplemented") }
-func (Provider) KillTendermint(_ context.Context, _ *e2e.Node) error      { panic("unimplemented") }
+func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
+	panic("unimplemented")
+}
+func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
+	panic("unimplemented")
+}
+func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
+	panic("unimplemented")
+}
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.

From 241c96a96e4616be9a370c786efde9dd6a04ada7 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:42:15 -0500
Subject: [PATCH 33/73] switch exec compose to docker pkg

---
 test/e2e/pkg/infra/docker/docker.go | 20 ++++++++++++++++++++
 test/e2e/runner/main.go             |  2 +-
 test/e2e/runner/perturb.go          | 11 ++++++-----
 test/e2e/runner/start.go            |  5 +++--
 4 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 7ba8ca3cd4d..f9d23927447 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -8,6 +8,7 @@ import (
 	"text/template"
 
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
 	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
 )
 
@@ -93,3 +94,22 @@ services:
 	}
 	return buf.Bytes(), nil
 }
+
+// ExecCompose runs a Docker Compose command for a testnet.
+func ExecCompose(dir string, args ...string) error {
+	return exec.Command(context.Background(), append(
+		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
+		args...)...)
+}
+
+// ExecComposeVerbose runs a Docker Compose command for a testnet and displays its output.
+func ExecComposeVerbose(dir string, args ...string) error {
+	return exec.CommandVerbose(context.Background(), append(
+		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
+		args...)...)
+}
+
+// ExecDocker runs a Docker command.
+func ExecDocker(args ...string) error {
+	return exec.Command(context.Background(), append([]string{"docker"}, args...)...)
+}
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index 33edf64be8a..20315f4aa06 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -211,7 +211,7 @@ func NewCLI() *CLI {
 		Short: "Stops the Docker testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			logger.Info("Stopping testnet")
-			return execCompose(cli.testnet.Dir, "down")
+			return docker.ExecCompose(cli.testnet.Dir, "down")
 		},
 	})
 
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index f47aa72bfc4..5180774d95e 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -7,6 +7,7 @@ import (
 	"github.com/tendermint/tendermint/libs/log"
 	rpctypes "github.com/tendermint/tendermint/rpc/core/types"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
 )
 
 // Perturbs a running testnet.
@@ -40,26 +41,26 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 
 	case e2e.PerturbationKill:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Killing node %v...", node.Name))
-		if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
 			return nil, err
 		}
-		if err := execCompose(testnet.Dir, "start", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "start", node.Name); err != nil {
 			return nil, err
 		}
 
 	case e2e.PerturbationPause:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Pausing node %v...", node.Name))
-		if err := execCompose(testnet.Dir, "pause", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "pause", node.Name); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "unpause", node.Name); err != nil {
 			return nil, err
 		}
 
 	case e2e.PerturbationRestart:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Restarting node %v...", node.Name))
-		if err := execCompose(testnet.Dir, "restart", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "restart", node.Name); err != nil {
 			return nil, err
 		}
 
diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go
index 0dab98f22d6..a3082e17d29 100644
--- a/test/e2e/runner/start.go
+++ b/test/e2e/runner/start.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
 )
 
 func Start(testnet *e2e.Testnet) error {
@@ -43,7 +44,7 @@ func Start(testnet *e2e.Testnet) error {
 	for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 {
 		node := nodeQueue[0]
 		nodeQueue = nodeQueue[1:]
-		if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}
 		if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
@@ -97,7 +98,7 @@ func Start(testnet *e2e.Testnet) error {
 
 		logger.Info("Starting catch up node", "node", node.Name, "height", node.StartAt)
 
-		if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
+		if err := docker.ExecCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}
 		status, err := waitForNode(node, node.StartAt, 3*time.Minute)

From 55c0e472c9ef09b1415fc19a97a044134400d17e Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:43:52 -0500
Subject: [PATCH 34/73] all exec switched to docker package

---
 test/e2e/pkg/infra/docker/docker.go | 4 ++--
 test/e2e/runner/cleanup.go          | 3 ++-
 test/e2e/runner/main.go             | 4 ++--
 test/e2e/runner/perturb.go          | 4 ++--
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index f9d23927447..a74f7b5b53f 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -109,7 +109,7 @@ func ExecComposeVerbose(dir string, args ...string) error {
 		args...)...)
 }
 
-// ExecDocker runs a Docker command.
-func ExecDocker(args ...string) error {
+// Exec runs a Docker command.
+func Exec(args ...string) error {
 	return exec.Command(context.Background(), append([]string{"docker"}, args...)...)
 }
diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go
index 74e84f7fc72..5df402be80e 100644
--- a/test/e2e/runner/cleanup.go
+++ b/test/e2e/runner/cleanup.go
@@ -10,6 +10,7 @@ import (
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
 )
 
 // Cleanup removes the Docker Compose containers and testnet directory.
@@ -71,7 +72,7 @@ func cleanupDir(dir string) error {
 	if err != nil {
 		return err
 	}
-	err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
+	err = docker.Exec("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
 		"tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/")
 	if err != nil {
 		return err
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index 20315f4aa06..51143c7871f 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -266,7 +266,7 @@ func NewCLI() *CLI {
 		Use:   "logs",
 		Short: "Shows the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return execComposeVerbose(cli.testnet.Dir, "logs")
+			return docker.ExecComposeVerbose(cli.testnet.Dir, "logs")
 		},
 	})
 
@@ -274,7 +274,7 @@ func NewCLI() *CLI {
 		Use:   "tail",
 		Short: "Tails the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return execComposeVerbose(cli.testnet.Dir, "logs", "--follow")
+			return docker.ExecComposeVerbose(cli.testnet.Dir, "logs", "--follow")
 		},
 	})
 
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index 5180774d95e..a08de233273 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -31,11 +31,11 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 	switch perturbation {
 	case e2e.PerturbationDisconnect:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Disconnecting node %v...", node.Name))
-		if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
+		if err := docker.Exec("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
+		if err := docker.Exec("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
 			return nil, err
 		}
 

From 6389d3d3f372e93b261e79f2643dfc9eb537e83c Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:44:11 -0500
Subject: [PATCH 35/73] remove runner exec

---
 test/e2e/runner/exec.go | 27 ---------------------------
 1 file changed, 27 deletions(-)
 delete mode 100644 test/e2e/runner/exec.go

diff --git a/test/e2e/runner/exec.go b/test/e2e/runner/exec.go
deleted file mode 100644
index eedc447336c..00000000000
--- a/test/e2e/runner/exec.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package main
-
-import (
-	"path/filepath"
-
-	"github.com/tendermint/tendermint/test/e2e/pkg/exec"
-	"golang.org/x/net/context"
-)
-
-// execCompose runs a Docker Compose command for a testnet.
-func execCompose(dir string, args ...string) error {
-	return exec.Command(context.Background(), append(
-		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
-		args...)...)
-}
-
-// execComposeVerbose runs a Docker Compose command for a testnet and displays its output.
-func execComposeVerbose(dir string, args ...string) error {
-	return exec.CommandVerbose(context.Background(), append(
-		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
-		args...)...)
-}
-
-// execDocker runs a Docker command.
-func execDocker(args ...string) error {
-	return exec.Command(context.Background(), append([]string{"docker"}, args...)...)
-}

From 6415af6b924eeee2a0ef817f42c5320a38f2d359 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:46:28 -0500
Subject: [PATCH 36/73] functionality in place for start kill terminate

---
 test/e2e/pkg/infra/docker/docker.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index a74f7b5b53f..24ffa675b52 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -35,13 +35,13 @@ func (p *Provider) Setup() error {
 	return nil
 }
 func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
-	panic("unimplemented")
+	return ExecCompose(p.Testnet.Dir, "start", n.Name)
 }
 func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
-	panic("unimplemented")
+	return ExecCompose(p.Testnet.Dir, "kill", "-s", "SIGTERM", n.Name)
 }
 func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
-	panic("unimplemented")
+	return ExecCompose(p.Testnet.Dir, "kill", "-s", "SIGKILL", n.Name)
 }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the

From be8d980f83d0fd56591d5d144bb9a5ab1b823379 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:48:56 -0500
Subject: [PATCH 37/73] exec compose takes context

---
 test/e2e/pkg/infra/docker/docker.go | 10 +++++-----
 test/e2e/runner/main.go             |  2 +-
 test/e2e/runner/perturb.go          | 11 ++++++-----
 test/e2e/runner/start.go            |  5 +++--
 4 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 24ffa675b52..80798e26e09 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -35,13 +35,13 @@ func (p *Provider) Setup() error {
 	return nil
 }
 func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
-	return ExecCompose(p.Testnet.Dir, "start", n.Name)
+	return ExecCompose(ctx, p.Testnet.Dir, "start", n.Name)
 }
 func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
-	return ExecCompose(p.Testnet.Dir, "kill", "-s", "SIGTERM", n.Name)
+	return ExecCompose(ctx, p.Testnet.Dir, "kill", "-s", "SIGTERM", n.Name)
 }
 func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
-	return ExecCompose(p.Testnet.Dir, "kill", "-s", "SIGKILL", n.Name)
+	return ExecCompose(ctx, p.Testnet.Dir, "kill", "-s", "SIGKILL", n.Name)
 }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
@@ -96,8 +96,8 @@ services:
 }
 
 // ExecCompose runs a Docker Compose command for a testnet.
-func ExecCompose(dir string, args ...string) error {
-	return exec.Command(context.Background(), append(
+func ExecCompose(ctx context.Context, dir string, args ...string) error {
+	return exec.Command(ctx, append(
 		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
 		args...)...)
 }
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index 51143c7871f..bcea17956cd 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -211,7 +211,7 @@ func NewCLI() *CLI {
 		Short: "Stops the Docker testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			logger.Info("Stopping testnet")
-			return docker.ExecCompose(cli.testnet.Dir, "down")
+			return docker.ExecCompose(context.Background(), cli.testnet.Dir, "down") 
 		},
 	})
 
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index a08de233273..bab66680b5f 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -41,26 +42,26 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 
 	case e2e.PerturbationKill:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Killing node %v...", node.Name))
-		if err := docker.ExecCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
 			return nil, err
 		}
-		if err := docker.ExecCompose(testnet.Dir, "start", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "start", node.Name); err != nil {
 			return nil, err
 		}
 
 	case e2e.PerturbationPause:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Pausing node %v...", node.Name))
-		if err := docker.ExecCompose(testnet.Dir, "pause", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "pause", node.Name); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := docker.ExecCompose(testnet.Dir, "unpause", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "unpause", node.Name); err != nil {
 			return nil, err
 		}
 
 	case e2e.PerturbationRestart:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Restarting node %v...", node.Name))
-		if err := docker.ExecCompose(testnet.Dir, "restart", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "restart", node.Name); err != nil {
 			return nil, err
 		}
 
diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go
index a3082e17d29..08dab035f24 100644
--- a/test/e2e/runner/start.go
+++ b/test/e2e/runner/start.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"sort"
 	"time"
@@ -44,7 +45,7 @@ func Start(testnet *e2e.Testnet) error {
 	for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 {
 		node := nodeQueue[0]
 		nodeQueue = nodeQueue[1:]
-		if err := docker.ExecCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}
 		if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
@@ -98,7 +99,7 @@ func Start(testnet *e2e.Testnet) error {
 
 		logger.Info("Starting catch up node", "node", node.Name, "height", node.StartAt)
 
-		if err := docker.ExecCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
+		if err := docker.ExecCompose(context.Background(), testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}
 		status, err := waitForNode(node, node.StartAt, 3*time.Minute)

From eb46fdc720bb88aa99d58cc33580bd5e31226bca Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:49:56 -0500
Subject: [PATCH 38/73] compose verbose takes context

---
 test/e2e/pkg/infra/docker/docker.go | 4 ++--
 test/e2e/runner/main.go             | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 80798e26e09..19a2934aaff 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -103,8 +103,8 @@ func ExecCompose(ctx context.Context, dir string, args ...string) error {
 }
 
 // ExecComposeVerbose runs a Docker Compose command for a testnet and displays its output.
-func ExecComposeVerbose(dir string, args ...string) error {
-	return exec.CommandVerbose(context.Background(), append(
+func ExecComposeVerbose(ctx context.Context, dir string, args ...string) error {
+	return exec.CommandVerbose(ctx, append(
 		[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
 		args...)...)
 }
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index bcea17956cd..1c384008083 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -266,7 +266,7 @@ func NewCLI() *CLI {
 		Use:   "logs",
 		Short: "Shows the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return docker.ExecComposeVerbose(cli.testnet.Dir, "logs")
+			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs") 
 		},
 	})
 
@@ -274,7 +274,7 @@ func NewCLI() *CLI {
 		Use:   "tail",
 		Short: "Tails the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return docker.ExecComposeVerbose(cli.testnet.Dir, "logs", "--follow")
+			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs", "--follow") 
 		},
 	})
 

From 9e4a24681c9045d417f3447eeb844e7baabcfa2b Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Tue, 29 Nov 2022 13:50:39 -0500
Subject: [PATCH 39/73] docker.exec takes context

---
 test/e2e/pkg/infra/docker/docker.go | 4 ++--
 test/e2e/runner/cleanup.go          | 4 ++--
 test/e2e/runner/perturb.go          | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 19a2934aaff..03c19cacfb1 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -110,6 +110,6 @@ func ExecComposeVerbose(ctx context.Context, dir string, args ...string) error {
 }
 
 // Exec runs a Docker command.
-func Exec(args ...string) error {
-	return exec.Command(context.Background(), append([]string{"docker"}, args...)...)
+func Exec(ctx context.Context, args ...string) error {
+	return exec.Command(ctx, append([]string{"docker"}, args...)...)
 }
diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go
index 5df402be80e..2cc7f9b68a5 100644
--- a/test/e2e/runner/cleanup.go
+++ b/test/e2e/runner/cleanup.go
@@ -72,8 +72,8 @@ func cleanupDir(dir string) error {
 	if err != nil {
 		return err
 	}
-	err = docker.Exec("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
-		"tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/")
+	err = docker.Exec(context.Background(), "run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
+		"tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") 
 	if err != nil {
 		return err
 	}
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index bab66680b5f..dfd0ff8b72b 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -32,11 +32,11 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 	switch perturbation {
 	case e2e.PerturbationDisconnect:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Disconnecting node %v...", node.Name))
-		if err := docker.Exec("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
+		if err := docker.Exec(context.Background(), "network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name) ; err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := docker.Exec("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
+		if err := docker.Exec(context.Background(), "network", "connect", testnet.Name+"_"+testnet.Name, node.Name) ; err != nil {
 			return nil, err
 		}
 

From eec848b4084e98f34b4c2a0c6ca8fa3ffff3dbf1 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:00:41 -0500
Subject: [PATCH 40/73] add create node to infra provider

---
 test/e2e/pkg/infra/docker/docker.go | 4 ++++
 test/e2e/pkg/infra/provider.go      | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 03c19cacfb1..a177002985b 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -34,6 +34,10 @@ func (p *Provider) Setup() error {
 	}
 	return nil
 }
+
+func (p Provider) CreateNode(ctx context.Context, n *e2e.Node) error {
+	return ExecCompose(ctx, p.Testnet.Dir, "create", n.Name)
+}
 func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
 	return ExecCompose(ctx, p.Testnet.Dir, "start", n.Name)
 }
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index bff50eb6d20..0fdcc86baa8 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -14,6 +14,7 @@ type Provider interface {
 	// provider during testnet setup.
 	Setup() error
 
+	CreateNode(context.Context, *e2e.Node) error
 	StartTendermint(context.Context, *e2e.Node) error
 	KillTendermint(context.Context, *e2e.Node) error
 	TerminateTendermint(context.Context, *e2e.Node) error
@@ -26,6 +27,7 @@ type NoopProvider struct {
 }
 
 func (NoopProvider) Setup() error                                             { return nil }
+func (NoopProvider) CreateNode(_ context.Context, _ *e2e.Node) error          { return nil }
 func (NoopProvider) StartTendermint(_ context.Context, _ *e2e.Node) error     { return nil }
 func (NoopProvider) TerminateTendermint(_ context.Context, _ *e2e.Node) error { return nil }
 func (NoopProvider) KillTendermint(_ context.Context, _ *e2e.Node) error      { return nil }

From 0cafe8930144eb1f676d9736487147dd4cf261da Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:06:30 -0500
Subject: [PATCH 41/73] start and create calls replaced with infra provider

---
 test/e2e/runner/main.go  | 12 ++++++------
 test/e2e/runner/start.go | 18 ++++++++++++++----
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index 1c384008083..a2a8a3b104d 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -112,7 +112,7 @@ func NewCLI() *CLI {
 				chLoadResult <- err
 			}()
 
-			if err := Start(cli.testnet); err != nil {
+			if err := Start(cli.testnet, cli.infp); err != nil {
 				return err
 			}
 
@@ -186,7 +186,7 @@ func NewCLI() *CLI {
 			if err != nil {
 				return err
 			}
-			return Start(cli.testnet)
+			return Start(cli.testnet, cli.infp)
 		},
 	})
 
@@ -211,7 +211,7 @@ func NewCLI() *CLI {
 		Short: "Stops the Docker testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			logger.Info("Stopping testnet")
-			return docker.ExecCompose(context.Background(), cli.testnet.Dir, "down") 
+			return docker.ExecCompose(context.Background(), cli.testnet.Dir, "down")
 		},
 	})
 
@@ -266,7 +266,7 @@ func NewCLI() *CLI {
 		Use:   "logs",
 		Short: "Shows the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs") 
+			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs")
 		},
 	})
 
@@ -274,7 +274,7 @@ func NewCLI() *CLI {
 		Use:   "tail",
 		Short: "Tails the testnet logs",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs", "--follow") 
+			return docker.ExecComposeVerbose(context.Background(), cli.testnet.Dir, "logs", "--follow")
 		},
 	})
 
@@ -309,7 +309,7 @@ Does not run any perturbations.
 				chLoadResult <- err
 			}()
 
-			if err := Start(cli.testnet); err != nil {
+			if err := Start(cli.testnet, cli.infp); err != nil {
 				return err
 			}
 
diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go
index 08dab035f24..4a8a521bcdb 100644
--- a/test/e2e/runner/start.go
+++ b/test/e2e/runner/start.go
@@ -8,10 +8,10 @@ import (
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
-	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
 )
 
-func Start(testnet *e2e.Testnet) error {
+func Start(testnet *e2e.Testnet, p infra.Provider) error {
 	if len(testnet.Nodes) == 0 {
 		return fmt.Errorf("no nodes in testnet")
 	}
@@ -45,7 +45,12 @@ func Start(testnet *e2e.Testnet) error {
 	for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 {
 		node := nodeQueue[0]
 		nodeQueue = nodeQueue[1:]
-		if err := docker.ExecCompose(context.Background(), testnet.Dir, "up", "-d", node.Name); err != nil {
+		err := p.CreateNode(context.Background(), node)
+		if err != nil {
+			return err
+		}
+		err = p.StartTendermint(context.Background(), node)
+		if err != nil {
 			return err
 		}
 		if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
@@ -99,7 +104,12 @@ func Start(testnet *e2e.Testnet) error {
 
 		logger.Info("Starting catch up node", "node", node.Name, "height", node.StartAt)
 
-		if err := docker.ExecCompose(context.Background(), testnet.Dir, "up", "-d", node.Name); err != nil {
+		err := p.CreateNode(context.Background(), node)
+		if err != nil {
+			return err
+		}
+		err = p.StartTendermint(context.Background(), node)
+		if err != nil {
 			return err
 		}
 		status, err := waitForNode(node, node.StartAt, 3*time.Minute)

From db7b4a0e3278a18bc5f351f35f6aab67f82666fd Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:41:28 -0500
Subject: [PATCH 42/73] commit exec pkg

---
 test/e2e/pkg/exec/exec.go | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 test/e2e/pkg/exec/exec.go

diff --git a/test/e2e/pkg/exec/exec.go b/test/e2e/pkg/exec/exec.go
new file mode 100644
index 00000000000..9dcd793844a
--- /dev/null
+++ b/test/e2e/pkg/exec/exec.go
@@ -0,0 +1,34 @@
+package exec
+
+import (
+	"context"
+	"fmt"
+	"os"
+	osexec "os/exec"
+)
+
+// Command executes a shell command.
+func Command(ctx context.Context, args ...string) error {
+	// nolint: gosec
+	// G204: Subprocess launched with a potential tainted input or cmd arguments
+	cmd := osexec.CommandContext(ctx, args[0], args[1:]...)
+	out, err := cmd.CombinedOutput()
+	switch err := err.(type) {
+	case nil:
+		return nil
+	case *osexec.ExitError:
+		return fmt.Errorf("failed to run %q:\n%v", args, string(out))
+	default:
+		return err
+	}
+}
+
+// CommandVerbose executes a shell command while displaying its output.
+func CommandVerbose(ctx context.Context, args ...string) error {
+	// nolint: gosec
+	// G204: Subprocess launched with a potential tainted input or cmd arguments
+	cmd := osexec.CommandContext(ctx, args[0], args[1:]...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}

From e009a2c7eed4eaf3a03f9d42dc8a33a38393070e Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:41:38 -0500
Subject: [PATCH 43/73] add ssh pkg

---
 test/e2e/pkg/ssh/ssh.go | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 test/e2e/pkg/ssh/ssh.go

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
new file mode 100644
index 00000000000..bd2619d3561
--- /dev/null
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -0,0 +1,33 @@
+package ssh
+
+import (
+	"os"
+	"path/filepath"
+
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+)
+
+func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
+	c, err := ssh.Dial("tcp", addr, cfg)
+	if err != nil {
+		return err
+	}
+	s, err := c.NewSession()
+	defer s.Close()
+	err = s.Run(cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func NewClientConfig(key string) (*ssh.ClientConfig, error) {
+	hkc, err := knownhosts.New(filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts"))
+	if err != nil {
+		return nil, err
+	}
+	return &ssh.ClientConfig{
+		HostKeyCallback: hkc,
+	}, nil
+}

From b0be713546ef10ace9ff425dde365348a0e9924b Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:41:48 -0500
Subject: [PATCH 44/73] add digital ocean package

---
 .../pkg/infra/digitalocean/digitalocean.go    | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 test/e2e/pkg/infra/digitalocean/digitalocean.go

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
new file mode 100644
index 00000000000..3529a78b839
--- /dev/null
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -0,0 +1,44 @@
+package digitalocean
+
+import (
+	"context"
+	"fmt"
+
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
+	e2essh "github.com/tendermint/tendermint/test/e2e/pkg/ssh"
+	"golang.org/x/crypto/ssh"
+)
+
+const (
+	sshPort     = 22
+	testappName = "testappd"
+)
+
+var _ infra.Provider = &Provider{}
+
+// Provider implements a docker-compose backed infrastructure provider.
+type Provider struct {
+	Testnet            *e2e.Testnet
+	InfrastructureData e2e.InfrastructureData
+	SSHConfig          *ssh.ClientConfig
+}
+
+// Noop currently. Setup is performed externally to the e2e test tool.
+func (p *Provider) Setup() error {
+	return nil
+}
+
+// Noop currently. Node creation is currently performed externally to the e2e test tool.
+func (p Provider) CreateNode(ctx context.Context, n *e2e.Node) error {
+	return nil
+}
+func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort), fmt.Sprintf("systemctl start %s", testappName))
+}
+func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort), fmt.Sprintf("systemctl -s SIGTERM %s", testappName))
+}
+func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort), fmt.Sprintf("systemctl -s SIGKILL %s", testappName))
+}

From b7fccf01382c2af3414031f1bf8efd00a8cc056c Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 12:46:00 -0500
Subject: [PATCH 45/73] add client creation logic

---
 test/e2e/pkg/ssh/ssh.go | 14 +++++++++++++-
 test/e2e/runner/main.go | 21 +++++++++++++++++++--
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index bd2619d3561..f672f0cae4d 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -22,12 +22,24 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 	return nil
 }
 
-func NewClientConfig(key string) (*ssh.ClientConfig, error) {
+func NewClientConfig(keyFile string) (*ssh.ClientConfig, error) {
 	hkc, err := knownhosts.New(filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts"))
 	if err != nil {
 		return nil, err
 	}
+	key, err := os.ReadFile(keyFile)
+	if err != nil {
+		return nil, err
+	}
+	signer, err := ssh.ParsePrivateKey(key)
+	if err != nil {
+		return nil, err
+	}
 	return &ssh.ClientConfig{
 		HostKeyCallback: hkc,
+		Auth: []ssh.AuthMethod{
+			ssh.PublicKeys(signer),
+		},
+		HostKeyAlgorithms: []string{ssh.KeyAlgoED25519},
 	}, nil
 }
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index a2a8a3b104d..bb5a0cc2f07 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"path/filepath"
 	"strconv"
 
 	"github.com/spf13/cobra"
@@ -13,7 +14,9 @@ import (
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 	"github.com/tendermint/tendermint/test/e2e/pkg/infra"
+	"github.com/tendermint/tendermint/test/e2e/pkg/infra/digitalocean"
 	"github.com/tendermint/tendermint/test/e2e/pkg/infra/docker"
+	e2essh "github.com/tendermint/tendermint/test/e2e/pkg/ssh"
 )
 
 const randomSeed = 2308084734268
@@ -85,9 +88,23 @@ func NewCLI() *CLI {
 			}
 
 			cli.testnet = testnet
-			cli.infp = &infra.NoopProvider{}
-			if inft == "docker" {
+			switch inft {
+			case "docker":
 				cli.infp = &docker.Provider{Testnet: testnet}
+			case "digitalocean":
+
+				// TMP: hardcoded key
+				cfg, err := e2essh.NewClientConfig(filepath.Join(os.Getenv("HOME"), ".ssh", "PVT_KEY"))
+				if err != nil {
+					return err
+				}
+				cli.infp = &digitalocean.Provider{
+					Testnet:            testnet,
+					InfrastructureData: ifd,
+					SSHConfig:          cfg,
+				}
+			default:
+				cli.infp = &infra.NoopProvider{}
 			}
 			return nil
 		},

From de20e3d2243c94c8a02eda80908144f69d1df7fe Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 14:12:12 -0500
Subject: [PATCH 46/73] use ssh agent to auth to remote

---
 test/e2e/pkg/infrastructure.go |  4 ++++
 test/e2e/pkg/ssh/ssh.go        | 18 +++++++++---------
 test/e2e/pkg/testnet.go        |  8 ++++----
 test/e2e/runner/main.go        | 14 +++++++++-----
 test/e2e/runner/start.go       |  6 +++---
 5 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/test/e2e/pkg/infrastructure.go b/test/e2e/pkg/infrastructure.go
index 2fc0e4bac6e..582ed491aec 100644
--- a/test/e2e/pkg/infrastructure.go
+++ b/test/e2e/pkg/infrastructure.go
@@ -38,6 +38,7 @@ type InfrastructureData struct {
 // one of the nodes in the testnet.
 type InstanceData struct {
 	IPAddress net.IP `json:"ip_address"`
+	Port      uint32 `json:"port"`
 }
 
 func NewDockerInfrastructureData(m Manifest) (InfrastructureData, error) {
@@ -49,6 +50,8 @@ func NewDockerInfrastructureData(m Manifest) (InfrastructureData, error) {
 	if err != nil {
 		return InfrastructureData{}, fmt.Errorf("invalid IP network address %q: %w", netAddress, err)
 	}
+
+	portGen := newPortGenerator(proxyPortFirst)
 	ipGen := newIPGenerator(ipNet)
 	ifd := InfrastructureData{
 		Provider:  "docker",
@@ -58,6 +61,7 @@ func NewDockerInfrastructureData(m Manifest) (InfrastructureData, error) {
 	for name := range m.Nodes {
 		ifd.Instances[name] = InstanceData{
 			IPAddress: ipGen.Next(),
+			Port:      portGen.Next(),
 		}
 	}
 	return ifd, nil
diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index f672f0cae4d..ea74e23d526 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 
 	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/agent"
 	"golang.org/x/crypto/ssh/knownhosts"
 )
 
@@ -22,24 +23,23 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 	return nil
 }
 
-func NewClientConfig(keyFile string) (*ssh.ClientConfig, error) {
+func NewClientConfig(ac agent.ExtendedAgent) (*ssh.ClientConfig, error) {
 	hkc, err := knownhosts.New(filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts"))
 	if err != nil {
 		return nil, err
 	}
-	key, err := os.ReadFile(keyFile)
+	signers, err := ac.Signers()
 	if err != nil {
 		return nil, err
 	}
-	signer, err := ssh.ParsePrivateKey(key)
-	if err != nil {
-		return nil, err
+	am := make([]ssh.AuthMethod, 0, len(signers))
+	for _, signer := range signers {
+		am = append(am, ssh.PublicKeys(signer))
 	}
 	return &ssh.ClientConfig{
-		HostKeyCallback: hkc,
-		Auth: []ssh.AuthMethod{
-			ssh.PublicKeys(signer),
-		},
+		User:              "root",
+		HostKeyCallback:   hkc,
+		Auth:              am,
 		HostKeyAlgorithms: []string{ssh.KeyAlgoED25519},
 	}, nil
 }
diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go
index 8d0d674db1f..76a7afaa587 100644
--- a/test/e2e/pkg/testnet.go
+++ b/test/e2e/pkg/testnet.go
@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -112,7 +113,6 @@ type Node struct {
 func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Testnet, error) {
 	dir := strings.TrimSuffix(fname, filepath.Ext(fname))
 	keyGen := newKeyGenerator(randomSeed)
-	proxyPortGen := newPortGenerator(proxyPortFirst)
 	_, ipNet, err := net.ParseCIDR(ifd.Network)
 	if err != nil {
 		return nil, fmt.Errorf("invalid IP network address %q: %w", ifd.Network, err)
@@ -175,7 +175,7 @@ func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Test
 			PrivvalKey:       keyGen.Generate(manifest.KeyType),
 			NodeKey:          keyGen.Generate("ed25519"),
 			IP:               ind.IPAddress,
-			ProxyPort:        proxyPortGen.Next(),
+			ProxyPort:        ind.Port,
 			Mode:             ModeValidator,
 			SyncApp:          nodeManifest.SyncApp,
 			Database:         "goleveldb",
@@ -320,7 +320,7 @@ func (n Node) Validate(testnet Testnet) error {
 			return fmt.Errorf("local port %v must be >1024", n.ProxyPort)
 		}
 		for _, peer := range testnet.Nodes {
-			if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort {
+			if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort && bytes.Equal(peer.IP, n.IP) {
 				return fmt.Errorf("peer %q also has local port %v", peer.Name, n.ProxyPort)
 			}
 		}
@@ -460,7 +460,7 @@ func (n Node) AddressRPC() string {
 
 // Client returns an RPC client for a node.
 func (n Node) Client() (*rpchttp.HTTP, error) {
-	return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort), "/websocket")
+	return rpchttp.New(fmt.Sprintf("http://%s:%v", n.IP, n.ProxyPort), "/websocket")
 }
 
 // Stateless returns true if the node is either a seed node or a light node
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index bb5a0cc2f07..caf0bf7f68b 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -5,11 +5,12 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
+	"net"
 	"os"
-	"path/filepath"
 	"strconv"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/crypto/ssh/agent"
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
@@ -91,10 +92,13 @@ func NewCLI() *CLI {
 			switch inft {
 			case "docker":
 				cli.infp = &docker.Provider{Testnet: testnet}
-			case "digitalocean":
-
-				// TMP: hardcoded key
-				cfg, err := e2essh.NewClientConfig(filepath.Join(os.Getenv("HOME"), ".ssh", "PVT_KEY"))
+			case "digital-ocean":
+				c, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")) // TODO(williambanfield): Ensure this is a consistent location for the authentication socket of the agent.
+				if err != nil {
+					return err
+				}
+				ac := agent.NewClient(c)
+				cfg, err := e2essh.NewClientConfig(ac)
 				if err != nil {
 					return err
 				}
diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go
index 4a8a521bcdb..72ee3db6d41 100644
--- a/test/e2e/runner/start.go
+++ b/test/e2e/runner/start.go
@@ -56,7 +56,7 @@ func Start(testnet *e2e.Testnet, p infra.Provider) error {
 		if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
 			return err
 		}
-		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort))
+		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://%s:%v", node.Name, node.IP, node.ProxyPort))
 	}
 
 	networkHeight := testnet.InitialHeight
@@ -116,8 +116,8 @@ func Start(testnet *e2e.Testnet, p infra.Provider) error {
 		if err != nil {
 			return err
 		}
-		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://127.0.0.1:%v at height %v",
-			node.Name, node.ProxyPort, status.SyncInfo.LatestBlockHeight))
+		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://%s:%v at height %v",
+			node.Name, node.IP, node.ProxyPort, status.SyncInfo.LatestBlockHeight))
 	}
 
 	return nil

From 1a2afa62c5d64af4102ba8d80b18fc4c4619429b Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 17:48:54 -0500
Subject: [PATCH 47/73] plump infra data into tests

---
 test/e2e/pkg/infrastructure.go |  6 ++++++
 test/e2e/runner/main.go        |  6 ++++--
 test/e2e/runner/test.go        | 12 +++++++++++-
 test/e2e/tests/e2e_test.go     | 21 +++++++++++++++++++--
 4 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/test/e2e/pkg/infrastructure.go b/test/e2e/pkg/infrastructure.go
index 582ed491aec..041e1a8d0c4 100644
--- a/test/e2e/pkg/infrastructure.go
+++ b/test/e2e/pkg/infrastructure.go
@@ -17,6 +17,7 @@ const (
 // InfrastructureData contains the relevant information for a set of existing
 // infrastructure that is to be used for running a testnet.
 type InfrastructureData struct {
+	path string
 
 	// Provider is the name of infrastructure provider backing the testnet.
 	// For example, 'docker' if it is running locally in a docker network or
@@ -41,6 +42,10 @@ type InstanceData struct {
 	Port      uint32 `json:"port"`
 }
 
+func (i InfrastructureData) Path() string {
+	return i.path
+}
+
 func NewDockerInfrastructureData(m Manifest) (InfrastructureData, error) {
 	netAddress := dockerIPv4CIDR
 	if m.IPv6 {
@@ -80,5 +85,6 @@ func InfrastructureDataFromFile(p string) (InfrastructureData, error) {
 	if ifd.Network == "" {
 		ifd.Network = globalIPv4CIDR
 	}
+	ifd.path = p
 	return ifd, nil
 }
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index caf0bf7f68b..c2d0567f218 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -34,6 +34,7 @@ type CLI struct {
 	testnet  *e2e.Testnet
 	preserve bool
 	infp     infra.Provider
+	ifd      e2e.InfrastructureData
 }
 
 // NewCLI sets up the CLI.
@@ -82,6 +83,7 @@ func NewCLI() *CLI {
 			default:
 				return fmt.Errorf("unknown infrastructure type '%s'", inft)
 			}
+			cli.ifd = ifd
 
 			testnet, err := e2e.LoadTestnet(m, file, ifd)
 			if err != nil {
@@ -166,7 +168,7 @@ func NewCLI() *CLI {
 			if err := Wait(cli.testnet, 5); err != nil { // wait for network to settle before tests
 				return err
 			}
-			if err := Test(cli.testnet); err != nil {
+			if err := Test(cli.testnet, cli.ifd); err != nil {
 				return err
 			}
 			if !cli.preserve {
@@ -271,7 +273,7 @@ func NewCLI() *CLI {
 		Use:   "test",
 		Short: "Runs test cases against a running testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return Test(cli.testnet)
+			return Test(cli.testnet, cli.ifd)
 		},
 	})
 
diff --git a/test/e2e/runner/test.go b/test/e2e/runner/test.go
index 4671a81d262..819d5b9ac08 100644
--- a/test/e2e/runner/test.go
+++ b/test/e2e/runner/test.go
@@ -9,13 +9,23 @@ import (
 )
 
 // Test runs test cases under tests/
-func Test(testnet *e2e.Testnet) error {
+func Test(testnet *e2e.Testnet, ifd e2e.InfrastructureData) error {
 	logger.Info("Running tests in ./tests/...")
 
 	err := os.Setenv("E2E_MANIFEST", testnet.File)
 	if err != nil {
 		return err
 	}
+	if p := ifd.Path(); p != "" {
+		err = os.Setenv("INFRASTRUCTURE_DATA", p)
+		if err != nil {
+			return err
+		}
+	}
+	err = os.Setenv("INFRASTRUCTURE_TYPE", ifd.Provider)
+	if err != nil {
+		return err
+	}
 
 	return exec.CommandVerbose(context.Background(), "go", "test", "-count", "1", "./tests/...")
 }
diff --git a/test/e2e/tests/e2e_test.go b/test/e2e/tests/e2e_test.go
index 5df3309d418..8e5ee9379d7 100644
--- a/test/e2e/tests/e2e_test.go
+++ b/test/e2e/tests/e2e_test.go
@@ -73,7 +73,14 @@ func loadTestnet(t *testing.T) e2e.Testnet {
 	if !filepath.IsAbs(manifestFile) {
 		manifestFile = filepath.Join("..", manifestFile)
 	}
-
+	ifdType := os.Getenv("INFRASTRUCTURE_DATA")
+	ifdFile := os.Getenv("INFRASTRUCTURE_FILE")
+	if ifdFile == "" && ifdType != "docker" {
+		t.Skip("INFRASTRUCTURE_DATA not set and INFRASTRUCTURE_TYPE is not docker")
+	}
+	if !filepath.IsAbs(ifdFile) {
+		manifestFile = filepath.Join("..", manifestFile)
+	}
 	testnetCacheMtx.Lock()
 	defer testnetCacheMtx.Unlock()
 	if testnet, ok := testnetCache[manifestFile]; ok {
@@ -81,7 +88,17 @@ func loadTestnet(t *testing.T) e2e.Testnet {
 	}
 	m, err := e2e.LoadManifest(manifestFile)
 	require.NoError(t, err)
-	ifd, err := e2e.NewDockerInfrastructureData(m)
+
+	var ifd e2e.InfrastructureData
+	switch ifdType {
+	case "docker":
+		ifd, err = e2e.NewDockerInfrastructureData(m)
+		require.NoError(t, err)
+	case "digital-ocean":
+		ifd, err = e2e.InfrastructureDataFromFile(manifestFile)
+		require.NoError(t, err)
+	default:
+	}
 	require.NoError(t, err)
 
 	testnet, err := e2e.LoadTestnet(m, manifestFile, ifd)

From 9c8c6c4cffcb8990b8be372d0a63e4714c4852a0 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 18:02:16 -0500
Subject: [PATCH 48/73] move agent socket dial to client connection creation

---
 test/e2e/pkg/ssh/ssh.go | 8 +++++++-
 test/e2e/runner/main.go | 9 +--------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index ea74e23d526..07be10ccc05 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -1,6 +1,7 @@
 package ssh
 
 import (
+	"net"
 	"os"
 	"path/filepath"
 
@@ -23,7 +24,12 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 	return nil
 }
 
-func NewClientConfig(ac agent.ExtendedAgent) (*ssh.ClientConfig, error) {
+func NewClientConfig() (*ssh.ClientConfig, error) {
+	c, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
+	if err != nil {
+		return nil, err
+	}
+	ac := agent.NewClient(c)
 	hkc, err := knownhosts.New(filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts"))
 	if err != nil {
 		return nil, err
diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
index c2d0567f218..e599b730933 100644
--- a/test/e2e/runner/main.go
+++ b/test/e2e/runner/main.go
@@ -5,12 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
-	"net"
 	"os"
 	"strconv"
 
 	"github.com/spf13/cobra"
-	"golang.org/x/crypto/ssh/agent"
 
 	"github.com/tendermint/tendermint/libs/log"
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
@@ -95,12 +93,7 @@ func NewCLI() *CLI {
 			case "docker":
 				cli.infp = &docker.Provider{Testnet: testnet}
 			case "digital-ocean":
-				c, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")) // TODO(williambanfield): Ensure this is a consistent location for the authentication socket of the agent.
-				if err != nil {
-					return err
-				}
-				ac := agent.NewClient(c)
-				cfg, err := e2essh.NewClientConfig(ac)
+				cfg, err := e2essh.NewClientConfig()
 				if err != nil {
 					return err
 				}

From 3cfb90fb5b24ef894da3c6eeaf88c955b99b024b Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 18:05:10 -0500
Subject: [PATCH 49/73] error if auth sock not defined

---
 test/e2e/pkg/ssh/ssh.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index 07be10ccc05..08a5324e65b 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -1,6 +1,7 @@
 package ssh
 
 import (
+	"errors"
 	"net"
 	"os"
 	"path/filepath"
@@ -25,7 +26,11 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 }
 
 func NewClientConfig() (*ssh.ClientConfig, error) {
-	c, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
+	ss := os.Getenv("SSH_AUTH_SOCK")
+	if ss == "" {
+		return nil, errors.New("SSH_AUTH_SOCK environment variable is empty. Is the ssh-agent running?")
+	}
+	c, err := net.Dial("unix", ss)
 	if err != nil {
 		return nil, err
 	}

From 5a6b3abe27213f768b963b698c99b38b4cec2b7b Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 18:37:35 -0500
Subject: [PATCH 50/73] allow unknown hosts to be accessed

---
 test/e2e/pkg/ssh/ssh.go | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index 08a5324e65b..5246ada838f 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -2,6 +2,8 @@ package ssh
 
 import (
 	"errors"
+	"fmt"
+	"log"
 	"net"
 	"os"
 	"path/filepath"
@@ -49,8 +51,28 @@ func NewClientConfig() (*ssh.ClientConfig, error) {
 	}
 	return &ssh.ClientConfig{
 		User:              "root",
-		HostKeyCallback:   hkc,
+		HostKeyCallback:   hkcWrapper(hkc),
 		Auth:              am,
 		HostKeyAlgorithms: []string{ssh.KeyAlgoED25519},
 	}, nil
 }
+
+func hkcWrapper(hkc ssh.HostKeyCallback) ssh.HostKeyCallback {
+	return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+		err := hkc(hostname, remote, key)
+		if err == nil {
+			return nil
+		}
+		ke := &knownhosts.KeyError{}
+		if errors.As(err, &ke) && len(ke.Want) == 0 {
+			h, _, err := net.SplitHostPort(hostname)
+			if err != nil {
+				panic(fmt.Errorf("hostname incorrectly formatted: %w", err))
+			}
+			log.Printf("ignoring knownhosts error for unknown host: %s", h)
+			return nil
+		}
+
+		return err
+	}
+}

From 8ec6013b1d6db59c514f2d50cc1dab9e24f91e1b Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 30 Nov 2022 19:00:45 -0500
Subject: [PATCH 51/73] return session error

---
 test/e2e/pkg/ssh/ssh.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index 5246ada838f..18f02f651b0 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -19,6 +19,9 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 		return err
 	}
 	s, err := c.NewSession()
+	if err != nil {
+		return err
+	}
 	defer s.Close()
 	err = s.Run(cmd)
 	if err != nil {

From 59f93e9035be4748bde84a7fb3def09bb958833e Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Thu, 1 Dec 2022 11:21:51 -0500
Subject: [PATCH 52/73] close connection in ssh exec

---
 test/e2e/pkg/ssh/ssh.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index 18f02f651b0..cd56bbf240f 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -18,6 +18,7 @@ func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
 	if err != nil {
 		return err
 	}
+	defer c.Close()
 	s, err := c.NewSession()
 	if err != nil {
 		return err

From 2bcc84efd4b725846158114911d328d30a517479 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Thu, 1 Dec 2022 11:30:19 -0500
Subject: [PATCH 53/73] gofmt

---
 test/e2e/runner/cleanup.go | 2 +-
 test/e2e/runner/perturb.go | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go
index 2cc7f9b68a5..33a7f9623d4 100644
--- a/test/e2e/runner/cleanup.go
+++ b/test/e2e/runner/cleanup.go
@@ -73,7 +73,7 @@ func cleanupDir(dir string) error {
 		return err
 	}
 	err = docker.Exec(context.Background(), "run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir),
-		"tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") 
+		"tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/")
 	if err != nil {
 		return err
 	}
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index dfd0ff8b72b..13edc22e1e4 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -32,11 +32,11 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 	switch perturbation {
 	case e2e.PerturbationDisconnect:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Disconnecting node %v...", node.Name))
-		if err := docker.Exec(context.Background(), "network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name) ; err != nil {
+		if err := docker.Exec(context.Background(), "network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := docker.Exec(context.Background(), "network", "connect", testnet.Name+"_"+testnet.Name, node.Name) ; err != nil {
+		if err := docker.Exec(context.Background(), "network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
 			return nil, err
 		}
 

From ca78052a7e4ba09104aab253196d94d1c0d2412d Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Thu, 1 Dec 2022 11:31:30 -0500
Subject: [PATCH 54/73] use IP.Equal instead of bytes.Equal

---
 test/e2e/pkg/testnet.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go
index 76a7afaa587..e5c58d3c8b4 100644
--- a/test/e2e/pkg/testnet.go
+++ b/test/e2e/pkg/testnet.go
@@ -1,7 +1,6 @@
 package e2e
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -320,7 +319,7 @@ func (n Node) Validate(testnet Testnet) error {
 			return fmt.Errorf("local port %v must be >1024", n.ProxyPort)
 		}
 		for _, peer := range testnet.Nodes {
-			if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort && bytes.Equal(peer.IP, n.IP) {
+			if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort && peer.IP.Equal(n.IP) {
 				return fmt.Errorf("peer %q also has local port %v", peer.Name, n.ProxyPort)
 			}
 		}

From 644a93a75984d17ceb6b49b0c4a124cf6fa79a01 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Thu, 1 Dec 2022 11:31:51 -0500
Subject: [PATCH 55/73] correct the nolint comment format

---
 test/e2e/pkg/exec/exec.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/e2e/pkg/exec/exec.go b/test/e2e/pkg/exec/exec.go
index 9dcd793844a..968ca6658e1 100644
--- a/test/e2e/pkg/exec/exec.go
+++ b/test/e2e/pkg/exec/exec.go
@@ -9,7 +9,7 @@ import (
 
 // Command executes a shell command.
 func Command(ctx context.Context, args ...string) error {
-	// nolint: gosec
+	//nolint: gosec
 	// G204: Subprocess launched with a potential tainted input or cmd arguments
 	cmd := osexec.CommandContext(ctx, args[0], args[1:]...)
 	out, err := cmd.CombinedOutput()
@@ -25,7 +25,7 @@ func Command(ctx context.Context, args ...string) error {
 
 // CommandVerbose executes a shell command while displaying its output.
 func CommandVerbose(ctx context.Context, args ...string) error {
-	// nolint: gosec
+	//nolint: gosec
 	// G204: Subprocess launched with a potential tainted input or cmd arguments
 	cmd := osexec.CommandContext(ctx, args[0], args[1:]...)
 	cmd.Stdout = os.Stdout

From d5119c20f70b1e432133563da2e268190b5a8040 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Mon, 5 Dec 2022 10:13:17 -0500
Subject: [PATCH 56/73] remove backup for INFRASTRUCTURE_DATA missing

---
 test/e2e/tests/e2e_test.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/test/e2e/tests/e2e_test.go b/test/e2e/tests/e2e_test.go
index 8e5ee9379d7..e9ea3e40384 100644
--- a/test/e2e/tests/e2e_test.go
+++ b/test/e2e/tests/e2e_test.go
@@ -78,9 +78,6 @@ func loadTestnet(t *testing.T) e2e.Testnet {
 	if ifdFile == "" && ifdType != "docker" {
 		t.Skip("INFRASTRUCTURE_DATA not set and INFRASTRUCTURE_TYPE is not docker")
 	}
-	if !filepath.IsAbs(ifdFile) {
-		manifestFile = filepath.Join("..", manifestFile)
-	}
 	testnetCacheMtx.Lock()
 	defer testnetCacheMtx.Unlock()
 	if testnet, ok := testnetCache[manifestFile]; ok {
@@ -95,7 +92,7 @@ func loadTestnet(t *testing.T) e2e.Testnet {
 		ifd, err = e2e.NewDockerInfrastructureData(m)
 		require.NoError(t, err)
 	case "digital-ocean":
-		ifd, err = e2e.InfrastructureDataFromFile(manifestFile)
+		ifd, err = e2e.InfrastructureDataFromFile(ifdFile)
 		require.NoError(t, err)
 	default:
 	}

From 143109eb283b9a88ae8b4b78ce4f8067bb99d050 Mon Sep 17 00:00:00 2001
From: William Banfield <4561443+williambanfield@users.noreply.github.com>
Date: Mon, 5 Dec 2022 10:15:43 -0500
Subject: [PATCH 57/73] Update test/e2e/pkg/infra/digitalocean/digitalocean.go

Co-authored-by: Thane Thomson <connect@thanethomson.com>
---
 test/e2e/pkg/infra/digitalocean/digitalocean.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index 3529a78b839..14a148a4133 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -17,7 +17,7 @@ const (
 
 var _ infra.Provider = &Provider{}
 
-// Provider implements a docker-compose backed infrastructure provider.
+// Provider implements a DigitalOcean-backed infrastructure provider.
 type Provider struct {
 	Testnet            *e2e.Testnet
 	InfrastructureData e2e.InfrastructureData

From 4dd0ff5621e8568dc407e7256bd52f8fb0c2ddc1 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Mon, 5 Dec 2022 10:17:46 -0500
Subject: [PATCH 58/73] remove hostkeyalgorithm setting

---
 test/e2e/pkg/ssh/ssh.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index cd56bbf240f..14ca37738b2 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -54,10 +54,9 @@ func NewClientConfig() (*ssh.ClientConfig, error) {
 		am = append(am, ssh.PublicKeys(signer))
 	}
 	return &ssh.ClientConfig{
-		User:              "root",
-		HostKeyCallback:   hkcWrapper(hkc),
-		Auth:              am,
-		HostKeyAlgorithms: []string{ssh.KeyAlgoED25519},
+		User:            "root",
+		HostKeyCallback: hkcWrapper(hkc),
+		Auth:            am,
 	}, nil
 }
 

From ac539b1f3dfb20ff5b1c9420c5ac11f4db337456 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Mon, 5 Dec 2022 11:25:01 -0500
Subject: [PATCH 59/73] rename IP node field

---
 .../pkg/infra/digitalocean/digitalocean.go    |  6 ++---
 test/e2e/pkg/ssh/ssh.go                       |  4 ++++
 test/e2e/pkg/testnet.go                       | 22 +++++++++----------
 test/e2e/runner/start.go                      |  4 ++--
 test/e2e/tests/net_test.go                    |  2 +-
 5 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index 14a148a4133..f5bd0d2e840 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -34,11 +34,11 @@ func (p Provider) CreateNode(ctx context.Context, n *e2e.Node) error {
 	return nil
 }
 func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
-	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort), fmt.Sprintf("systemctl start %s", testappName))
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.InternalIP, sshPort), fmt.Sprintf("systemctl start %s", testappName))
 }
 func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
-	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort), fmt.Sprintf("systemctl -s SIGTERM %s", testappName))
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.InternalIP, sshPort), fmt.Sprintf("systemctl -s SIGTERM %s", testappName))
 }
 func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
-	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.IP, sshPort), fmt.Sprintf("systemctl -s SIGKILL %s", testappName))
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.InternalIP, sshPort), fmt.Sprintf("systemctl -s SIGKILL %s", testappName))
 }
diff --git a/test/e2e/pkg/ssh/ssh.go b/test/e2e/pkg/ssh/ssh.go
index 14ca37738b2..d129e9c8092 100644
--- a/test/e2e/pkg/ssh/ssh.go
+++ b/test/e2e/pkg/ssh/ssh.go
@@ -7,6 +7,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"time"
 
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/agent"
@@ -14,10 +15,13 @@ import (
 )
 
 func Exec(cfg *ssh.ClientConfig, addr, cmd string) error {
+	start := time.Now()
 	c, err := ssh.Dial("tcp", addr, cfg)
 	if err != nil {
 		return err
 	}
+	after := time.Since(start)
+	fmt.Printf("Time: %s\n", after)
 	defer c.Close()
 	s, err := c.NewSession()
 	if err != nil {
diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go
index e5c58d3c8b4..c09dc6e9024 100644
--- a/test/e2e/pkg/testnet.go
+++ b/test/e2e/pkg/testnet.go
@@ -84,7 +84,7 @@ type Node struct {
 	SyncApp          bool // Should we use a synchronized app with an unsynchronized local client?
 	PrivvalKey       crypto.PrivKey
 	NodeKey          crypto.PrivKey
-	IP               net.IP
+	InternalIP       net.IP
 	ProxyPort        uint32
 	StartAt          int64
 	BlockSync        string
@@ -173,7 +173,7 @@ func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Test
 			Testnet:          testnet,
 			PrivvalKey:       keyGen.Generate(manifest.KeyType),
 			NodeKey:          keyGen.Generate("ed25519"),
-			IP:               ind.IPAddress,
+			InternalIP:       ind.IPAddress,
 			ProxyPort:        ind.Port,
 			Mode:             ModeValidator,
 			SyncApp:          nodeManifest.SyncApp,
@@ -308,18 +308,18 @@ func (n Node) Validate(testnet Testnet) error {
 	if n.Name == "" {
 		return errors.New("node has no name")
 	}
-	if n.IP == nil {
+	if n.InternalIP == nil {
 		return errors.New("node has no IP address")
 	}
-	if !testnet.IP.Contains(n.IP) {
-		return fmt.Errorf("node IP %v is not in testnet network %v", n.IP, testnet.IP)
+	if !testnet.IP.Contains(n.InternalIP) {
+		return fmt.Errorf("node IP %v is not in testnet network %v", n.InternalIP, testnet.IP)
 	}
 	if n.ProxyPort > 0 {
 		if n.ProxyPort <= 1024 {
 			return fmt.Errorf("local port %v must be >1024", n.ProxyPort)
 		}
 		for _, peer := range testnet.Nodes {
-			if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort && peer.IP.Equal(n.IP) {
+			if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort && peer.InternalIP.Equal(n.InternalIP) {
 				return fmt.Errorf("peer %q also has local port %v", peer.Name, n.ProxyPort)
 			}
 		}
@@ -435,8 +435,8 @@ func (t Testnet) HasPerturbations() bool {
 
 // Address returns a P2P endpoint address for the node.
 func (n Node) AddressP2P(withID bool) string {
-	ip := n.IP.String()
-	if n.IP.To4() == nil {
+	ip := n.InternalIP.String()
+	if n.InternalIP.To4() == nil {
 		// IPv6 addresses must be wrapped in [] to avoid conflict with : port separator
 		ip = fmt.Sprintf("[%v]", ip)
 	}
@@ -449,8 +449,8 @@ func (n Node) AddressP2P(withID bool) string {
 
 // Address returns an RPC endpoint address for the node.
 func (n Node) AddressRPC() string {
-	ip := n.IP.String()
-	if n.IP.To4() == nil {
+	ip := n.InternalIP.String()
+	if n.InternalIP.To4() == nil {
 		// IPv6 addresses must be wrapped in [] to avoid conflict with : port separator
 		ip = fmt.Sprintf("[%v]", ip)
 	}
@@ -459,7 +459,7 @@ func (n Node) AddressRPC() string {
 
 // Client returns an RPC client for a node.
 func (n Node) Client() (*rpchttp.HTTP, error) {
-	return rpchttp.New(fmt.Sprintf("http://%s:%v", n.IP, n.ProxyPort), "/websocket")
+	return rpchttp.New(fmt.Sprintf("http://%s:%v", n.InternalIP, n.ProxyPort), "/websocket")
 }
 
 // Stateless returns true if the node is either a seed node or a light node
diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go
index 72ee3db6d41..5353334ea55 100644
--- a/test/e2e/runner/start.go
+++ b/test/e2e/runner/start.go
@@ -56,7 +56,7 @@ func Start(testnet *e2e.Testnet, p infra.Provider) error {
 		if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
 			return err
 		}
-		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://%s:%v", node.Name, node.IP, node.ProxyPort))
+		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://%s:%v", node.Name, node.InternalIP, node.ProxyPort))
 	}
 
 	networkHeight := testnet.InitialHeight
@@ -117,7 +117,7 @@ func Start(testnet *e2e.Testnet, p infra.Provider) error {
 			return err
 		}
 		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://%s:%v at height %v",
-			node.Name, node.IP, node.ProxyPort, status.SyncInfo.LatestBlockHeight))
+			node.Name, node.InternalIP, node.ProxyPort, status.SyncInfo.LatestBlockHeight))
 	}
 
 	return nil
diff --git a/test/e2e/tests/net_test.go b/test/e2e/tests/net_test.go
index 1ca43fa0556..00f50d3f3e3 100644
--- a/test/e2e/tests/net_test.go
+++ b/test/e2e/tests/net_test.go
@@ -34,7 +34,7 @@ func TestNet_Peers(t *testing.T) {
 		for _, peerInfo := range netInfo.Peers {
 			peer := node.Testnet.LookupNode(peerInfo.NodeInfo.Moniker)
 			require.NotNil(t, peer, "unknown node %v", peerInfo.NodeInfo.Moniker)
-			require.Equal(t, peer.IP.String(), peerInfo.RemoteIP,
+			require.Equal(t, peer.InternalIP.String(), peerInfo.RemoteIP,
 				"unexpected IP address for peer %v", peer.Name)
 			seen[peerInfo.NodeInfo.Moniker] = true
 		}

From 8e3f608770a6438aa51bc39e2b0f2ff5460d6555 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Mon, 5 Dec 2022 11:28:42 -0500
Subject: [PATCH 60/73] add external address field

---
 test/e2e/pkg/testnet.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go
index c09dc6e9024..e273cc1a359 100644
--- a/test/e2e/pkg/testnet.go
+++ b/test/e2e/pkg/testnet.go
@@ -85,6 +85,7 @@ type Node struct {
 	PrivvalKey       crypto.PrivKey
 	NodeKey          crypto.PrivKey
 	InternalIP       net.IP
+	ExternalIP       net.IP
 	ProxyPort        uint32
 	StartAt          int64
 	BlockSync        string
@@ -161,6 +162,7 @@ func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Test
 		nodeNames = append(nodeNames, name)
 	}
 	sort.Strings(nodeNames)
+	localHostIP := net.ParseIP("127.0.0.1")
 
 	for _, name := range nodeNames {
 		nodeManifest := manifest.Nodes[name]
@@ -168,12 +170,17 @@ func LoadTestnet(manifest Manifest, fname string, ifd InfrastructureData) (*Test
 		if !ok {
 			return nil, fmt.Errorf("information for node '%s' missing from infrastucture data", name)
 		}
+		extIP := localHostIP
+		if ifd.Provider != "docker" {
+			extIP = ind.IPAddress
+		}
 		node := &Node{
 			Name:             name,
 			Testnet:          testnet,
 			PrivvalKey:       keyGen.Generate(manifest.KeyType),
 			NodeKey:          keyGen.Generate("ed25519"),
 			InternalIP:       ind.IPAddress,
+			ExternalIP:       extIP,
 			ProxyPort:        ind.Port,
 			Mode:             ModeValidator,
 			SyncApp:          nodeManifest.SyncApp,
@@ -459,7 +466,7 @@ func (n Node) AddressRPC() string {
 
 // Client returns an RPC client for a node.
 func (n Node) Client() (*rpchttp.HTTP, error) {
-	return rpchttp.New(fmt.Sprintf("http://%s:%v", n.InternalIP, n.ProxyPort), "/websocket")
+	return rpchttp.New(fmt.Sprintf("http://%s:%v", n.ExternalIP, n.ProxyPort), "/websocket")
 }
 
 // Stateless returns true if the node is either a seed node or a light node

From e22b61ab9d2b69775949e3c04d23a38b9c3b0849 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Mon, 5 Dec 2022 11:34:20 -0500
Subject: [PATCH 61/73] use external IP in digital ocean

---
 test/e2e/pkg/infra/digitalocean/digitalocean.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index f5bd0d2e840..3fbadc964f5 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -34,11 +34,11 @@ func (p Provider) CreateNode(ctx context.Context, n *e2e.Node) error {
 	return nil
 }
 func (p Provider) StartTendermint(ctx context.Context, n *e2e.Node) error {
-	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.InternalIP, sshPort), fmt.Sprintf("systemctl start %s", testappName))
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.ExternalIP, sshPort), fmt.Sprintf("systemctl start %s", testappName))
 }
 func (p Provider) TerminateTendermint(ctx context.Context, n *e2e.Node) error {
-	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.InternalIP, sshPort), fmt.Sprintf("systemctl -s SIGTERM %s", testappName))
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.ExternalIP, sshPort), fmt.Sprintf("systemctl -s SIGTERM %s", testappName))
 }
 func (p Provider) KillTendermint(ctx context.Context, n *e2e.Node) error {
-	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.InternalIP, sshPort), fmt.Sprintf("systemctl -s SIGKILL %s", testappName))
+	return e2essh.Exec(p.SSHConfig, fmt.Sprintf("%s:%d", n.ExternalIP, sshPort), fmt.Sprintf("systemctl -s SIGKILL %s", testappName))
 }

From 64a65c75f18901600e4a529a236b943a642f40cd Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Thu, 8 Dec 2022 15:27:57 -0500
Subject: [PATCH 62/73] use correct variable in e2e compose

---
 test/e2e/pkg/infra/docker/docker.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index d5c37a524a4..e83c0b4a49c 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -85,7 +85,7 @@ services:
     - ./{{ .Name }}:/tendermint
     networks:
       {{ $.Name }}:
-        ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }}
+        ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .InternalIP }}
 
 {{end}}`)
 	if err != nil {

From 384f3aebad5e11ecded3c0ef40dbc9683ef1dfa6 Mon Sep 17 00:00:00 2001
From: Sergio Mena <sergio@informal.systems>
Date: Wed, 17 May 2023 14:29:07 +0200
Subject: [PATCH 63/73] fix node name for the case '*_u' (upgraded)

---
 test/e2e/pkg/infra/docker/docker.go | 8 ++++----
 test/e2e/pkg/infra/provider.go      | 5 +++--
 test/e2e/runner/perturb.go          | 4 ++--
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index 708ab12fb53..fdbe6396b9c 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -45,11 +45,11 @@ func (p Provider) StartNodes(ctx context.Context, nodes ...*e2e.Node) error {
 func (p Provider) StopTestnet(ctx context.Context) error {
 	return ExecCompose(ctx, p.Testnet.Dir, "down")
 }
-func (p Provider) Connect(ctx context.Context, n *e2e.Node) error {
-	return Exec(ctx, "network", "connect", p.Testnet.Name+"_"+p.Testnet.Name, n.Name)
+func (p Provider) Connect(ctx context.Context, name string) error {
+	return Exec(ctx, "network", "connect", p.Testnet.Name+"_"+p.Testnet.Name, name)
 }
-func (p Provider) Disconnect(ctx context.Context, n *e2e.Node) error {
-	return Exec(ctx, "network", "disconnect", p.Testnet.Name+"_"+p.Testnet.Name, n.Name)
+func (p Provider) Disconnect(ctx context.Context, name string) error {
+	return Exec(ctx, "network", "disconnect", p.Testnet.Name+"_"+p.Testnet.Name, name)
 }
 
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index 181bfe8a84a..ca8623cf75a 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -22,10 +22,11 @@ type Provider interface {
 	// Stops the whole network
 	StopTestnet(context.Context) error
 
+	Connect(context.Context, string) error
+	Disconnect(context.Context, string) error
+
 	// Returns the the provider's infrastructure data
 	GetInfrastructureData() *e2e.InfrastructureData
-	Connect(context.Context, *e2e.Node) error
-	Disconnect(context.Context, *e2e.Node) error
 }
 
 type ProviderData struct {
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index bf32fd7bd07..c41e5022cd5 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -47,11 +47,11 @@ func PerturbNode(ctx context.Context, node *e2e.Node, perturbation e2e.Perturbat
 	switch perturbation {
 	case e2e.PerturbationDisconnect:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Disconnecting node %v...", node.Name))
-		if err := ifp.Disconnect(context.Background(), node); err != nil {
+		if err := ifp.Disconnect(context.Background(), name); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := ifp.Connect(context.Background(), node); err != nil {
+		if err := ifp.Connect(context.Background(), name); err != nil {
 			return nil, err
 		}
 

From 51db8ecf8df7fbe92cf3e3282a209dcc3e04f194 Mon Sep 17 00:00:00 2001
From: Sergio Mena <sergio@informal.systems>
Date: Wed, 17 May 2023 15:34:53 +0200
Subject: [PATCH 64/73] Adapt (dis)connection logic to ansible

---
 .../pkg/infra/digitalocean/digitalocean.go    | 96 +++++++++++++------
 test/e2e/pkg/infra/docker/docker.go           |  4 +-
 test/e2e/pkg/infra/provider.go                |  4 +-
 test/e2e/runner/perturb.go                    |  4 +-
 4 files changed, 73 insertions(+), 35 deletions(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index 48e6c4dbf03..f485356288a 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -24,14 +24,18 @@ func (p *Provider) Setup() error {
 	return nil
 }
 
-const ymlSystemd = "systemd-action.yml"
+const (
+	ymlSystemd = "systemd-action.yml"
+	ymlConnect = "connect-action.yml"
+)
 
 func (p Provider) StartNodes(ctx context.Context, nodes ...*e2e.Node) error {
 	nodeIPs := make([]string, len(nodes))
 	for i, n := range nodes {
 		nodeIPs[i] = n.ExternalIP.String()
 	}
-	if err := p.writePlaybook(ymlSystemd, true); err != nil {
+	playbook := ansibleSystemdBytes(true)
+	if err := p.writePlaybook(ymlSystemd, playbook); err != nil {
 		return err
 	}
 
@@ -43,27 +47,28 @@ func (p Provider) StopTestnet(ctx context.Context) error {
 		nodeIPs[i] = n.ExternalIP.String()
 	}
 
-	if err := p.writePlaybook(ymlSystemd, false); err != nil {
+	playbook := ansibleSystemdBytes(false)
+	if err := p.writePlaybook(ymlSystemd, playbook); err != nil {
 		return err
 	}
 	return execAnsible(ctx, p.Testnet.Dir, ymlSystemd, nodeIPs)
 }
-
-func (p Provider) Disconnect(ctx context.Context, n *e2e.Node) error {
-	return e2essh.MultiExec(p.SSHConfig, fmt.Sprintf("%s:%d", n.ExternalIP, sshPort),
-		"iptables -A INPUT -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset",
-		"iptables -A OUTPUT -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset",
-	)
+func (p Provider) Connect(ctx context.Context, _ string, ip string) error {
+	playbook := ansiblePerturbConnectionBytes(false)
+	if err := p.writePlaybook(ymlConnect, playbook); err != nil {
+		return err
+	}
+	return execAnsible(ctx, p.Testnet.Dir, ymlConnect, []string{ip})
 }
-func (p Provider) Connect(ctx context.Context, n *e2e.Node) error {
-	return e2essh.MultiExec(p.SSHConfig, fmt.Sprintf("%s:%d", n.ExternalIP, sshPort),
-		"iptables -D INPUT -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset",
-		"iptables -D OUTPUT -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset",
-	)
+func (p Provider) Disconnect(ctx context.Context, _ string, ip string) error {
+	playbook := ansiblePerturbConnectionBytes(true)
+	if err := p.writePlaybook(ymlConnect, playbook); err != nil {
+		return err
+	}
+	return execAnsible(ctx, p.Testnet.Dir, ymlConnect, []string{ip})
 }
 
-func (p Provider) writePlaybook(yaml string, starting bool) error {
-	playbook := ansibleSystemdBytes(starting)
+func (p Provider) writePlaybook(yaml, playbook string) error {
 	//nolint: gosec
 	// G306: Expect WriteFile permissions to be 0600 or less
 	err := os.WriteFile(filepath.Join(p.Testnet.Dir, yaml), []byte(playbook), 0o644)
@@ -73,25 +78,58 @@ func (p Provider) writePlaybook(yaml string, starting bool) error {
 	return nil
 }
 
-// file as bytes to be written out to disk.
-// ansibleStartBytes generates an Ansible playbook to start the network
-func ansibleSystemdBytes(starting bool) string {
-	startStop := "stopped"
-	if starting {
-		startStop = "started"
-	}
-	playbook := fmt.Sprintf(`- name: start/stop testapp
+const basePlaybook = `- name: start/stop testapp
   hosts: all
   gather_facts: yes
   vars:
     ansible_host_key_checking: false
 
   tasks:
-  - name: operate on the systemd-unit
-    ansible.builtin.systemd:
-      name: testappd
-      state: %s
-      enabled: yes`, startStop)
+`
+
+func ansibleAddTask(playbook, name, contents string) string {
+	return playbook + "  - name: " + name + "\n" + contents
+}
+
+func ansibleAddSystemdTask(playbook string, starting bool) string {
+	startStop := "stopped"
+	if starting {
+		startStop = "started"
+	}
+	contents := fmt.Sprintf(`    ansible.builtin.systemd:
+	  name: testappd
+	  state: %s
+	  enabled: yes`, startStop)
+
+	return ansibleAddTask(playbook, "operate on the systemd-unit", contents)
+}
+
+func ansibleAddShellTasks(playbook, name string, shells ...string) string {
+	for shell := range shells {
+		contents := fmt.Sprintf("    shell: \"%s\"\n", shell)
+		playbook = ansibleAddTask(playbook, name, contents)
+	}
+	return playbook
+}
+
+// file as bytes to be written out to disk.
+// ansibleStartBytes generates an Ansible playbook to start the network
+func ansibleSystemdBytes(starting bool) string {
+	return ansibleAddSystemdTask(basePlaybook, starting)
+}
+
+func ansiblePerturbConnectionBytes(disconnect bool) string {
+	disconnecting := "disconnect"
+	op := "-A"
+	if disconnect {
+		disconnecting = "reconnect"
+		op = "-D"
+	}
+	playbook := basePlaybook
+	for _, dir := range []string{"INPUT", "OUTPUT"} {
+		playbook = ansibleAddShellTasks(playbook, disconnecting+" node",
+			"iptables %s %s -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset", op, dir)
+	}
 	return playbook
 }
 
diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index fdbe6396b9c..c67294881ee 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -45,10 +45,10 @@ func (p Provider) StartNodes(ctx context.Context, nodes ...*e2e.Node) error {
 func (p Provider) StopTestnet(ctx context.Context) error {
 	return ExecCompose(ctx, p.Testnet.Dir, "down")
 }
-func (p Provider) Connect(ctx context.Context, name string) error {
+func (p Provider) Connect(ctx context.Context, name string, _ string) error {
 	return Exec(ctx, "network", "connect", p.Testnet.Name+"_"+p.Testnet.Name, name)
 }
-func (p Provider) Disconnect(ctx context.Context, name string) error {
+func (p Provider) Disconnect(ctx context.Context, name string, _ string) error {
 	return Exec(ctx, "network", "disconnect", p.Testnet.Name+"_"+p.Testnet.Name, name)
 }
 
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index ca8623cf75a..f4cc095a952 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -22,8 +22,8 @@ type Provider interface {
 	// Stops the whole network
 	StopTestnet(context.Context) error
 
-	Connect(context.Context, string) error
-	Disconnect(context.Context, string) error
+	Connect(context.Context, string, string) error
+	Disconnect(context.Context, string, string) error
 
 	// Returns the the provider's infrastructure data
 	GetInfrastructureData() *e2e.InfrastructureData
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index c41e5022cd5..e9f420dd887 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -47,11 +47,11 @@ func PerturbNode(ctx context.Context, node *e2e.Node, perturbation e2e.Perturbat
 	switch perturbation {
 	case e2e.PerturbationDisconnect:
 		logger.Info("perturb node", "msg", log.NewLazySprintf("Disconnecting node %v...", node.Name))
-		if err := ifp.Disconnect(context.Background(), name); err != nil {
+		if err := ifp.Disconnect(context.Background(), name, node.ExternalIP.String()); err != nil {
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := ifp.Connect(context.Background(), name); err != nil {
+		if err := ifp.Connect(context.Background(), name, node.ExternalIP.String()); err != nil {
 			return nil, err
 		}
 

From e6348f04f47d6ce1324dd458c89b213749365dec Mon Sep 17 00:00:00 2001
From: Sergio Mena <sergio@informal.systems>
Date: Wed, 17 May 2023 15:51:12 +0200
Subject: [PATCH 65/73] Fixes

---
 test/e2e/pkg/infra/digitalocean/digitalocean.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index f485356288a..6cedd7b923c 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -97,15 +97,15 @@ func ansibleAddSystemdTask(playbook string, starting bool) string {
 		startStop = "started"
 	}
 	contents := fmt.Sprintf(`    ansible.builtin.systemd:
-	  name: testappd
-	  state: %s
-	  enabled: yes`, startStop)
+      name: testappd
+      state: %s
+      enabled: yes`, startStop)
 
 	return ansibleAddTask(playbook, "operate on the systemd-unit", contents)
 }
 
 func ansibleAddShellTasks(playbook, name string, shells ...string) string {
-	for shell := range shells {
+	for _, shell := range shells {
 		contents := fmt.Sprintf("    shell: \"%s\"\n", shell)
 		playbook = ansibleAddTask(playbook, name, contents)
 	}

From c418e52f71b83f0043601b184636641e87a6cd90 Mon Sep 17 00:00:00 2001
From: Sergio Mena <sergio@informal.systems>
Date: Wed, 17 May 2023 16:11:50 +0200
Subject: [PATCH 66/73] Make check for upgrade provider-dependent

---
 test/e2e/pkg/infra/digitalocean/digitalocean.go |  5 +++++
 test/e2e/pkg/infra/docker/docker.go             | 15 +++++++++++++++
 test/e2e/pkg/infra/provider.go                  |  1 +
 test/e2e/runner/perturb.go                      |  9 +++------
 4 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index 6cedd7b923c..b05ffff6f00 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -68,6 +68,11 @@ func (p Provider) Disconnect(ctx context.Context, _ string, ip string) error {
 	return execAnsible(ctx, p.Testnet.Dir, ymlConnect, []string{ip})
 }
 
+func (p Provider) CheckUpgraded(ctx context.Context, node *e2e.Node) (string, bool, error) {
+	// Upgrade not supported yet by DO provider
+	return node.Name, false, nil
+}
+
 func (p Provider) writePlaybook(yaml, playbook string) error {
 	//nolint: gosec
 	// G306: Expect WriteFile permissions to be 0600 or less
diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index c67294881ee..d401e048c40 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -52,6 +52,21 @@ func (p Provider) Disconnect(ctx context.Context, name string, _ string) error {
 	return Exec(ctx, "network", "disconnect", p.Testnet.Name+"_"+p.Testnet.Name, name)
 }
 
+func (p Provider) CheckUpgraded(ctx context.Context, node *e2e.Node) (string, bool, error) {
+	testnet := node.Testnet
+	out, err := ExecComposeOutput(ctx, testnet.Dir, "ps", "-q", node.Name)
+	if err != nil {
+		return "", false, err
+	}
+	name := node.Name
+	upgraded := false
+	if len(out) == 0 {
+		name = name + "_u"
+		upgraded = true
+	}
+	return name, upgraded, nil
+}
+
 // dockerComposeBytes generates a Docker Compose config file for a testnet and returns the
 // file as bytes to be written out to disk.
 func dockerComposeBytes(testnet *e2e.Testnet) ([]byte, error) {
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index f4cc095a952..956fa1e1b27 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -27,6 +27,7 @@ type Provider interface {
 
 	// Returns the the provider's infrastructure data
 	GetInfrastructureData() *e2e.InfrastructureData
+	CheckUpgraded(context.Context, *e2e.Node) (string, bool, error)
 }
 
 type ProviderData struct {
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index e9f420dd887..a98aa43ea35 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -30,15 +30,12 @@ func Perturb(ctx context.Context, testnet *e2e.Testnet, ifp infra.Provider) erro
 // after recovering.
 func PerturbNode(ctx context.Context, node *e2e.Node, perturbation e2e.Perturbation, ifp infra.Provider) (*rpctypes.ResultStatus, error) {
 	testnet := node.Testnet
-	out, err := docker.ExecComposeOutput(context.Background(), testnet.Dir, "ps", "-q", node.Name)
+
+	name, upgraded, err := ifp.CheckUpgraded(ctx, node)
 	if err != nil {
 		return nil, err
 	}
-	name := node.Name
-	upgraded := false
-	if len(out) == 0 {
-		name = name + "_u"
-		upgraded = true
+	if upgraded {
 		logger.Info("perturb node", "msg",
 			log.NewLazySprintf("Node %v already upgraded, operating on alternate container %v",
 				node.Name, name))

From c14e083fde96706c9ca45ff18af6732daedb6c70 Mon Sep 17 00:00:00 2001
From: Sergio Mena <sergio@informal.systems>
Date: Wed, 17 May 2023 16:15:05 +0200
Subject: [PATCH 67/73] fix `iptables` formatting

---
 test/e2e/pkg/infra/digitalocean/digitalocean.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index b05ffff6f00..bd4b4c9291a 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -133,7 +133,7 @@ func ansiblePerturbConnectionBytes(disconnect bool) string {
 	playbook := basePlaybook
 	for _, dir := range []string{"INPUT", "OUTPUT"} {
 		playbook = ansibleAddShellTasks(playbook, disconnecting+" node",
-			"iptables %s %s -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset", op, dir)
+			fmt.Sprintf("iptables %s %s -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset", op, dir))
 	}
 	return playbook
 }

From 9f8c426e5db5f619ef76bba457f1310769090f92 Mon Sep 17 00:00:00 2001
From: Sergio Mena <sergio@informal.systems>
Date: Wed, 17 May 2023 16:37:32 +0200
Subject: [PATCH 68/73] Fix iptables command

---
 test/e2e/pkg/infra/digitalocean/digitalocean.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index bd4b4c9291a..3f96995da3d 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -83,7 +83,7 @@ func (p Provider) writePlaybook(yaml, playbook string) error {
 	return nil
 }
 
-const basePlaybook = `- name: start/stop testapp
+const basePlaybook = `- name: e2e custom playbook
   hosts: all
   gather_facts: yes
   vars:
@@ -133,7 +133,7 @@ func ansiblePerturbConnectionBytes(disconnect bool) string {
 	playbook := basePlaybook
 	for _, dir := range []string{"INPUT", "OUTPUT"} {
 		playbook = ansibleAddShellTasks(playbook, disconnecting+" node",
-			fmt.Sprintf("iptables %s %s -p tcp --destination-port 26656 -j REJECT --reject-with tcp-reset", op, dir))
+			fmt.Sprintf("iptables %s %s -p tcp --dport 26656 -j DROP", op, dir))
 	}
 	return playbook
 }

From f139ba23bfb83e3f8c48b8b997c241278bcc04a2 Mon Sep 17 00:00:00 2001
From: Sergio Mena <sergio@informal.systems>
Date: Wed, 17 May 2023 16:40:18 +0200
Subject: [PATCH 69/73] revert me

---
 test/e2e/pkg/exec/exec.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/e2e/pkg/exec/exec.go b/test/e2e/pkg/exec/exec.go
index b0a3228fac6..062eff9b578 100644
--- a/test/e2e/pkg/exec/exec.go
+++ b/test/e2e/pkg/exec/exec.go
@@ -31,6 +31,7 @@ func CommandOutput(ctx context.Context, args ...string) ([]byte, error) {
 
 // CommandVerbose executes a shell command while displaying its output.
 func CommandVerbose(ctx context.Context, args ...string) error {
+	fmt.Println("COMMAND:", args)
 	//nolint: gosec
 	// G204: Subprocess launched with a potential tainted input or cmd arguments
 	cmd := osexec.CommandContext(ctx, args[0], args[1:]...)

From 416dd2057d39934383cd30f7d57bd89d095e2669 Mon Sep 17 00:00:00 2001
From: Sergio Mena <sergio@informal.systems>
Date: Wed, 17 May 2023 16:51:37 +0200
Subject: [PATCH 70/73] Keep every playbook used in a different file for
 troubleshooting

---
 .../pkg/infra/digitalocean/digitalocean.go    | 32 ++++++++++++-------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index 3f96995da3d..9bc6a3888ff 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	e2e "github.com/cometbft/cometbft/test/e2e/pkg"
@@ -24,10 +25,13 @@ func (p *Provider) Setup() error {
 	return nil
 }
 
-const (
-	ymlSystemd = "systemd-action.yml"
-	ymlConnect = "connect-action.yml"
-)
+var ymlPlaybookSeq int
+
+func getNextPlaybookFilename() string {
+	const ymlPlaybookAction = "playbook-action"
+	ymlPlaybookSeq++
+	return ymlPlaybookAction + strconv.Itoa(ymlPlaybookSeq) + ".yml"
+}
 
 func (p Provider) StartNodes(ctx context.Context, nodes ...*e2e.Node) error {
 	nodeIPs := make([]string, len(nodes))
@@ -35,11 +39,12 @@ func (p Provider) StartNodes(ctx context.Context, nodes ...*e2e.Node) error {
 		nodeIPs[i] = n.ExternalIP.String()
 	}
 	playbook := ansibleSystemdBytes(true)
-	if err := p.writePlaybook(ymlSystemd, playbook); err != nil {
+	playbookFile := getNextPlaybookFilename()
+	if err := p.writePlaybook(playbookFile, playbook); err != nil {
 		return err
 	}
 
-	return execAnsible(ctx, p.Testnet.Dir, ymlSystemd, nodeIPs)
+	return execAnsible(ctx, p.Testnet.Dir, playbookFile, nodeIPs)
 }
 func (p Provider) StopTestnet(ctx context.Context) error {
 	nodeIPs := make([]string, len(p.Testnet.Nodes))
@@ -48,24 +53,27 @@ func (p Provider) StopTestnet(ctx context.Context) error {
 	}
 
 	playbook := ansibleSystemdBytes(false)
-	if err := p.writePlaybook(ymlSystemd, playbook); err != nil {
+	playbookFile := getNextPlaybookFilename()
+	if err := p.writePlaybook(playbookFile, playbook); err != nil {
 		return err
 	}
-	return execAnsible(ctx, p.Testnet.Dir, ymlSystemd, nodeIPs)
+	return execAnsible(ctx, p.Testnet.Dir, playbookFile, nodeIPs)
 }
 func (p Provider) Connect(ctx context.Context, _ string, ip string) error {
 	playbook := ansiblePerturbConnectionBytes(false)
-	if err := p.writePlaybook(ymlConnect, playbook); err != nil {
+	playbookFile := getNextPlaybookFilename()
+	if err := p.writePlaybook(playbookFile, playbook); err != nil {
 		return err
 	}
-	return execAnsible(ctx, p.Testnet.Dir, ymlConnect, []string{ip})
+	return execAnsible(ctx, p.Testnet.Dir, playbookFile, []string{ip})
 }
 func (p Provider) Disconnect(ctx context.Context, _ string, ip string) error {
 	playbook := ansiblePerturbConnectionBytes(true)
-	if err := p.writePlaybook(ymlConnect, playbook); err != nil {
+	playbookFile := getNextPlaybookFilename()
+	if err := p.writePlaybook(playbookFile, playbook); err != nil {
 		return err
 	}
-	return execAnsible(ctx, p.Testnet.Dir, ymlConnect, []string{ip})
+	return execAnsible(ctx, p.Testnet.Dir, playbookFile, []string{ip})
 }
 
 func (p Provider) CheckUpgraded(ctx context.Context, node *e2e.Node) (string, bool, error) {

From b6d4751ded4c30c3e3786850f0ee8f08c84bcc4e Mon Sep 17 00:00:00 2001
From: Sergio Mena <sergio@informal.systems>
Date: Wed, 17 May 2023 16:57:40 +0200
Subject: [PATCH 71/73] Fixed bug that was swapping disconnect and reconnect

---
 test/e2e/pkg/infra/digitalocean/digitalocean.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index 9bc6a3888ff..45c69070bf6 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -132,11 +132,11 @@ func ansibleSystemdBytes(starting bool) string {
 }
 
 func ansiblePerturbConnectionBytes(disconnect bool) string {
-	disconnecting := "disconnect"
-	op := "-A"
+	disconnecting := "reconnect"
+	op := "-D"
 	if disconnect {
-		disconnecting = "reconnect"
-		op = "-D"
+		disconnecting = "disconnect"
+		op = "-A"
 	}
 	playbook := basePlaybook
 	for _, dir := range []string{"INPUT", "OUTPUT"} {

From 49012a1a943491c931694cffe54fea00ad1122e0 Mon Sep 17 00:00:00 2001
From: Sergio Mena <sergio@informal.systems>
Date: Wed, 17 May 2023 18:46:40 +0200
Subject: [PATCH 72/73] Rename `Connect` to `Reconnect`

---
 test/e2e/pkg/exec/exec.go                       | 1 -
 test/e2e/pkg/infra/digitalocean/digitalocean.go | 8 ++++----
 test/e2e/pkg/infra/docker/docker.go             | 6 +++---
 test/e2e/pkg/infra/provider.go                  | 8 +++++++-
 test/e2e/runner/perturb.go                      | 2 +-
 5 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/test/e2e/pkg/exec/exec.go b/test/e2e/pkg/exec/exec.go
index 062eff9b578..b0a3228fac6 100644
--- a/test/e2e/pkg/exec/exec.go
+++ b/test/e2e/pkg/exec/exec.go
@@ -31,7 +31,6 @@ func CommandOutput(ctx context.Context, args ...string) ([]byte, error) {
 
 // CommandVerbose executes a shell command while displaying its output.
 func CommandVerbose(ctx context.Context, args ...string) error {
-	fmt.Println("COMMAND:", args)
 	//nolint: gosec
 	// G204: Subprocess launched with a potential tainted input or cmd arguments
 	cmd := osexec.CommandContext(ctx, args[0], args[1:]...)
diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index 45c69070bf6..e1337bcaa3b 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -59,16 +59,16 @@ func (p Provider) StopTestnet(ctx context.Context) error {
 	}
 	return execAnsible(ctx, p.Testnet.Dir, playbookFile, nodeIPs)
 }
-func (p Provider) Connect(ctx context.Context, _ string, ip string) error {
-	playbook := ansiblePerturbConnectionBytes(false)
+func (p Provider) Disconnect(ctx context.Context, _ string, ip string) error {
+	playbook := ansiblePerturbConnectionBytes(true)
 	playbookFile := getNextPlaybookFilename()
 	if err := p.writePlaybook(playbookFile, playbook); err != nil {
 		return err
 	}
 	return execAnsible(ctx, p.Testnet.Dir, playbookFile, []string{ip})
 }
-func (p Provider) Disconnect(ctx context.Context, _ string, ip string) error {
-	playbook := ansiblePerturbConnectionBytes(true)
+func (p Provider) Reconnect(ctx context.Context, _ string, ip string) error {
+	playbook := ansiblePerturbConnectionBytes(false)
 	playbookFile := getNextPlaybookFilename()
 	if err := p.writePlaybook(playbookFile, playbook); err != nil {
 		return err
diff --git a/test/e2e/pkg/infra/docker/docker.go b/test/e2e/pkg/infra/docker/docker.go
index d401e048c40..a38c995566b 100644
--- a/test/e2e/pkg/infra/docker/docker.go
+++ b/test/e2e/pkg/infra/docker/docker.go
@@ -45,12 +45,12 @@ func (p Provider) StartNodes(ctx context.Context, nodes ...*e2e.Node) error {
 func (p Provider) StopTestnet(ctx context.Context) error {
 	return ExecCompose(ctx, p.Testnet.Dir, "down")
 }
-func (p Provider) Connect(ctx context.Context, name string, _ string) error {
-	return Exec(ctx, "network", "connect", p.Testnet.Name+"_"+p.Testnet.Name, name)
-}
 func (p Provider) Disconnect(ctx context.Context, name string, _ string) error {
 	return Exec(ctx, "network", "disconnect", p.Testnet.Name+"_"+p.Testnet.Name, name)
 }
+func (p Provider) Reconnect(ctx context.Context, name string, _ string) error {
+	return Exec(ctx, "network", "connect", p.Testnet.Name+"_"+p.Testnet.Name, name)
+}
 
 func (p Provider) CheckUpgraded(ctx context.Context, node *e2e.Node) (string, bool, error) {
 	testnet := node.Testnet
diff --git a/test/e2e/pkg/infra/provider.go b/test/e2e/pkg/infra/provider.go
index 956fa1e1b27..ab63f87ba01 100644
--- a/test/e2e/pkg/infra/provider.go
+++ b/test/e2e/pkg/infra/provider.go
@@ -22,11 +22,17 @@ type Provider interface {
 	// Stops the whole network
 	StopTestnet(context.Context) error
 
-	Connect(context.Context, string, string) error
+	// Disconnects the node from the network
 	Disconnect(context.Context, string, string) error
 
+	// Reconnects the node to the network.
+	// This should only be called after Disconnect
+	Reconnect(context.Context, string, string) error
+
 	// Returns the the provider's infrastructure data
 	GetInfrastructureData() *e2e.InfrastructureData
+
+	// Checks whether the node has been upgraded in this run
 	CheckUpgraded(context.Context, *e2e.Node) (string, bool, error)
 }
 
diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go
index a98aa43ea35..3d26060561e 100644
--- a/test/e2e/runner/perturb.go
+++ b/test/e2e/runner/perturb.go
@@ -48,7 +48,7 @@ func PerturbNode(ctx context.Context, node *e2e.Node, perturbation e2e.Perturbat
 			return nil, err
 		}
 		time.Sleep(10 * time.Second)
-		if err := ifp.Connect(context.Background(), name, node.ExternalIP.String()); err != nil {
+		if err := ifp.Reconnect(context.Background(), name, node.ExternalIP.String()); err != nil {
 			return nil, err
 		}
 

From 49e439043e58a25d4d6054cfef3c0f7b41ba0ea9 Mon Sep 17 00:00:00 2001
From: Sergio Mena <sergio@informal.systems>
Date: Wed, 17 May 2023 18:59:06 +0200
Subject: [PATCH 73/73] Update test/e2e/pkg/infra/digitalocean/digitalocean.go

---
 test/e2e/pkg/infra/digitalocean/digitalocean.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/pkg/infra/digitalocean/digitalocean.go b/test/e2e/pkg/infra/digitalocean/digitalocean.go
index e1337bcaa3b..d9af0c25228 100644
--- a/test/e2e/pkg/infra/digitalocean/digitalocean.go
+++ b/test/e2e/pkg/infra/digitalocean/digitalocean.go
@@ -76,7 +76,7 @@ func (p Provider) Reconnect(ctx context.Context, _ string, ip string) error {
 	return execAnsible(ctx, p.Testnet.Dir, playbookFile, []string{ip})
 }
 
-func (p Provider) CheckUpgraded(ctx context.Context, node *e2e.Node) (string, bool, error) {
+func (p Provider) CheckUpgraded(_ context.Context, node *e2e.Node) (string, bool, error) {
 	// Upgrade not supported yet by DO provider
 	return node.Name, false, nil
 }