From 7ac87c9bc6a5e226af066eb78202d1ee0754abab Mon Sep 17 00:00:00 2001 From: Will Sackfield Date: Tue, 30 Jul 2024 17:40:38 -0400 Subject: [PATCH 1/3] Ping the docker daemon to check if it is up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Send a ping to the docker daemon directly using the golang docker client. * Give it 5 seconds to answer, if it doesn’t answer send a warning to the user. --- go.mod | 9 +++++++++ go.sum | 19 +++++++++++++++++++ pkg/image/build.go | 24 ++++++++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/go.mod b/go.mod index d61fe8595d..8eb10ab1b0 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect github.com/GaijinEntertainment/go-exhaustruct/v3 v3.2.0 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/OpenPeeDeeP/depguard/v2 v2.2.0 // indirect github.com/alecthomas/go-check-sumtype v0.1.4 // indirect github.com/alexkohler/nakedret/v2 v2.0.4 // indirect @@ -70,6 +71,7 @@ require ( github.com/daixiang0/gci v0.13.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect + github.com/distribution/reference v0.6.0 // indirect github.com/dnephin/pflag v1.0.7 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect @@ -78,11 +80,14 @@ require ( github.com/ettle/strcase v0.2.0 // indirect github.com/fatih/color v1.17.0 // indirect github.com/fatih/structtag v1.2.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/firefart/nonamedreturns v1.0.5 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect github.com/ghostiam/protogetter v0.3.6 // indirect github.com/go-critic/go-critic v0.11.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-toolsmith/astcast v1.1.0 // indirect @@ -212,6 +217,10 @@ require ( gitlab.com/bosi/decorder v0.4.2 // indirect go-simpler.org/musttag v0.12.2 // indirect go-simpler.org/sloglint v0.7.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect go.uber.org/multierr v1.6.0 // indirect diff --git a/go.sum b/go.sum index ade45e3c17..c325a6bd08 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,8 @@ github.com/GaijinEntertainment/go-exhaustruct/v3 v3.2.0 h1:sATXp1x6/axKxz2Gjxv8M github.com/GaijinEntertainment/go-exhaustruct/v3 v3.2.0/go.mod h1:Nl76DrGNJTA1KJ0LePKBw/vznBX1EHbAZX8mwjR82nI= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA= github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= @@ -141,6 +143,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2oNn0GkeZE= @@ -167,6 +171,8 @@ github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/firefart/nonamedreturns v1.0.5 h1:tM+Me2ZaXs8tfdDw3X6DOX++wMCOqzYUho6tUTYIdRA= github.com/firefart/nonamedreturns v1.0.5/go.mod h1:gHJjDqhGM4WyPt639SOZs+G89Ko7QKH5R5BhnO6xJhw= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -190,8 +196,13 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= @@ -661,6 +672,14 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= diff --git a/pkg/image/build.go b/pkg/image/build.go index 56d233b325..72ec50e264 100644 --- a/pkg/image/build.go +++ b/pkg/image/build.go @@ -2,12 +2,15 @@ package image import ( "bytes" + "context" "encoding/json" "fmt" "os" "os/exec" "path" + "time" + "github.com/docker/docker/client" "github.com/getkin/kin-openapi/openapi3" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -31,6 +34,8 @@ const bundledSchemaPy = ".cog/schema.py" func Build(cfg *config.Config, dir, imageName string, secrets []string, noCache, separateWeights bool, useCudaBaseImage string, progressOutput string, schemaFile string, dockerfileFile string, useCogBaseImage bool) error { console.Infof("Building Docker image from environment in cog.yaml as %s...", imageName) + ping() + // remove bundled schema files that may be left from previous builds _ = os.Remove(bundledSchemaFile) _ = os.Remove(bundledSchemaPy) @@ -360,3 +365,22 @@ func restoreDockerignore() error { return os.Rename(dockerignoreBackupPath, ".dockerignore") } + +func ping() { + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + fmt.Println("Failed to create docker client.") + return + } + + ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + _, err = cli.Ping(ctxTimeout) + + if err != nil { + fmt.Println("Failed to ping docker, please try restarting the docker daemon.") + // We swallow this error, for now this should just be a warning. + return + } +} From 9484d42257ec2a9acec531452185e38dddbd4d47 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Wed, 31 Jul 2024 03:19:35 -0700 Subject: [PATCH 2/3] Move ping from image to docker package Update ping to return error instead of printing Update ping to take context and duration arguments --- pkg/cli/build.go | 6 ++++++ pkg/docker/ping.go | 26 ++++++++++++++++++++++++++ pkg/image/build.go | 24 ------------------------ 3 files changed, 32 insertions(+), 24 deletions(-) create mode 100644 pkg/docker/ping.go diff --git a/pkg/cli/build.go b/pkg/cli/build.go index 812fa7f5dc..6e88dce841 100644 --- a/pkg/cli/build.go +++ b/pkg/cli/build.go @@ -4,11 +4,13 @@ import ( "fmt" "os" "strings" + "time" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/replicate/cog/pkg/config" + "github.com/replicate/cog/pkg/docker" "github.com/replicate/cog/pkg/image" "github.com/replicate/cog/pkg/util/console" ) @@ -45,6 +47,10 @@ func newBuildCommand() *cobra.Command { } func buildCommand(cmd *cobra.Command, args []string) error { + if err := docker.Ping(cmd.Context(), time.Second*5); err != nil { + return fmt.Errorf("Failed to ping docker, please try restarting the docker daemon: %w", err) + } + cfg, projectDir, err := config.GetConfig(projectDirFlag) if err != nil { return err diff --git a/pkg/docker/ping.go b/pkg/docker/ping.go new file mode 100644 index 0000000000..b3df9fb55e --- /dev/null +++ b/pkg/docker/ping.go @@ -0,0 +1,26 @@ +package docker + +import ( + "context" + "fmt" + "time" + + "github.com/docker/docker/client" +) + +func Ping(ctx context.Context, duration time.Duration) error { + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + return fmt.Errorf("failed to create docker client: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, duration) + defer cancel() + + _, err = cli.Ping(ctx) + if err != nil { + return fmt.Errorf("failed to ping docker, please try restarting the docker daemon: %w", err) + } + + return nil +} diff --git a/pkg/image/build.go b/pkg/image/build.go index 72ec50e264..56d233b325 100644 --- a/pkg/image/build.go +++ b/pkg/image/build.go @@ -2,15 +2,12 @@ package image import ( "bytes" - "context" "encoding/json" "fmt" "os" "os/exec" "path" - "time" - "github.com/docker/docker/client" "github.com/getkin/kin-openapi/openapi3" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -34,8 +31,6 @@ const bundledSchemaPy = ".cog/schema.py" func Build(cfg *config.Config, dir, imageName string, secrets []string, noCache, separateWeights bool, useCudaBaseImage string, progressOutput string, schemaFile string, dockerfileFile string, useCogBaseImage bool) error { console.Infof("Building Docker image from environment in cog.yaml as %s...", imageName) - ping() - // remove bundled schema files that may be left from previous builds _ = os.Remove(bundledSchemaFile) _ = os.Remove(bundledSchemaPy) @@ -365,22 +360,3 @@ func restoreDockerignore() error { return os.Rename(dockerignoreBackupPath, ".dockerignore") } - -func ping() { - cli, err := client.NewClientWithOpts(client.FromEnv) - if err != nil { - fmt.Println("Failed to create docker client.") - return - } - - ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - - _, err = cli.Ping(ctxTimeout) - - if err != nil { - fmt.Println("Failed to ping docker, please try restarting the docker daemon.") - // We swallow this error, for now this should just be a warning. - return - } -} From a1b5c7ae0f2e1d7a6cc850d0e495b10218d7ee2d Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Wed, 31 Jul 2024 03:19:56 -0700 Subject: [PATCH 3/3] Add ping checks to push, run, predict, and train subcommands --- pkg/cli/predict.go | 5 +++++ pkg/cli/push.go | 5 +++++ pkg/cli/run.go | 6 ++++++ pkg/cli/train.go | 5 +++++ 4 files changed, 21 insertions(+) diff --git a/pkg/cli/predict.go b/pkg/cli/predict.go index c106678d54..b79b030a1d 100644 --- a/pkg/cli/predict.go +++ b/pkg/cli/predict.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "syscall" + "time" "github.com/getkin/kin-openapi/openapi3" "github.com/mitchellh/go-homedir" @@ -65,6 +66,10 @@ func cmdPredict(cmd *cobra.Command, args []string) error { volumes := []docker.Volume{} gpus := gpusFlag + if err := docker.Ping(cmd.Context(), time.Second*5); err != nil { + return fmt.Errorf("Failed to ping docker, please try restarting the docker daemon: %w", err) + } + if len(args) == 0 { // Build image diff --git a/pkg/cli/push.go b/pkg/cli/push.go index 6e088b1c7f..c03f4bc89d 100644 --- a/pkg/cli/push.go +++ b/pkg/cli/push.go @@ -3,6 +3,7 @@ package cli import ( "fmt" "strings" + "time" "github.com/spf13/cobra" @@ -35,6 +36,10 @@ func newPushCommand() *cobra.Command { } func push(cmd *cobra.Command, args []string) error { + if err := docker.Ping(cmd.Context(), time.Second*5); err != nil { + return fmt.Errorf("Failed to ping docker, please try restarting the docker daemon: %w", err) + } + cfg, projectDir, err := config.GetConfig(projectDirFlag) if err != nil { return err diff --git a/pkg/cli/run.go b/pkg/cli/run.go index 17a31a1ad8..6e6be5bf76 100644 --- a/pkg/cli/run.go +++ b/pkg/cli/run.go @@ -1,9 +1,11 @@ package cli import ( + "fmt" "runtime" "strconv" "strings" + "time" "github.com/spf13/cobra" @@ -50,6 +52,10 @@ func newRunCommand() *cobra.Command { } func run(cmd *cobra.Command, args []string) error { + if err := docker.Ping(cmd.Context(), time.Second*5); err != nil { + return fmt.Errorf("Failed to ping docker, please try restarting the docker daemon: %w", err) + } + cfg, projectDir, err := config.GetConfig(projectDirFlag) if err != nil { return err diff --git a/pkg/cli/train.go b/pkg/cli/train.go index 22d0693540..3f8b4a141f 100644 --- a/pkg/cli/train.go +++ b/pkg/cli/train.go @@ -5,6 +5,7 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/spf13/cobra" @@ -50,6 +51,10 @@ Otherwise, it will build the model in the current directory and train it.`, } func cmdTrain(cmd *cobra.Command, args []string) error { + if err := docker.Ping(cmd.Context(), time.Second*5); err != nil { + return fmt.Errorf("Failed to ping docker, please try restarting the docker daemon: %w", err) + } + imageName := "" volumes := []docker.Volume{} gpus := gpusFlag