Open
Description
(Report based on moby/moby master @ 9267f34 - 2nd Oct 2018)
If the graphdrivers ApplyDiff function fails and we're using tarsplit, then the daemon will usually panic rather than return the error. It is a race condition, but I am able to hit it 50:50, probably slightly higher in testing.
This is what you'll see from the client as an example
Step 4/13 : ENV GOPATH=/go PATH=$PATH:/go/bin SRC=/go/src/github.com/Microsoft/opengcs
---> Running in d07950d0f9d8
Removing intermediate container d07950d0f9d8
unexpected EOF
And this is the stack-trace from the daemon log. The panic is from the deferred tsw.Close() in applyTar for the useTarSplit case in layer/layer_store.go
panic: runtime error: invalid memory address or nil pointer dereference
[signal 0xc0000005 code=0x0 addr=0x0 pc=0x6997a9]
goroutine 119 [running]:
compress/flate.(*Writer).Write(0x0, 0xc000a7c000, 0x2d1, 0x6c4, 0x6c4, 0xc0f57dddfe, 0x0)
C:/go/src/compress/flate/deflate.go:709 +0x29
compress/gzip.(*Writer).Write(0xc0006e0420, 0xc000a7c000, 0x2d1, 0x6c4, 0x3030100, 0x1c6ebc0, 0xc0001374a0)
C:/go/src/compress/gzip/gzip.go:196 +0xdd
encoding/json.(*Encoder).Encode(0xc000c029b0, 0x1c6ebc0, 0xc0001374a0, 0xc0001374a0, 0xc000850028)
C:/go/src/encoding/json/stream.go:225 +0x1d1
github.com/docker/docker/vendor/github.com/vbatts/tar-split/tar/storage.(*jsonPacker).AddEntry(0xc001067860, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc00060a200, 0x200, ...)
e:/go/src/github.com/docker/docker/vendor/github.com/vbatts/tar-split/tar/storage/packer.go:97 +0x1a2
github.com/docker/docker/vendor/github.com/vbatts/tar-split/tar/asm.NewInputTarStream.func1(0x2013800, 0xc0009511a0, 0xc000b6a068, 0x2012f60, 0xc001067860, 0x2012f40, 0x3063288)
e:/go/src/github.com/docker/docker/vendor/github.com/vbatts/tar-split/tar/asm/disassemble.go:74 +0x453
created by github.com/docker/docker/vendor/github.com/vbatts/tar-split/tar/asm.NewInputTarStream
e:/go/src/github.com/docker/docker/vendor/github.com/vbatts/tar-split/tar/asm/disassemble.go:45 +0x204
With GOTRACEBACK set ($env:GOTRACEBACK=1 in Powershell, change as appropriate for Linux):
2018-10-02 18:52:24.557676 I | http: panic serving //./pipe/docker_engine: runtime error: invalid memory address or nil pointer dereference
goroutine 15 [running]:
net/http.(*conn).serve.func1(0xc000b74000)
C:/go/src/net/http/server.go:1746 +0xd7
panic(0x1aafee0, 0x3011530)
C:/go/src/runtime/panic.go:513 +0x1c7
compress/flate.(*Writer).Close(0x0, 0xc001051b40, 0x40)
C:/go/src/compress/flate/deflate.go:729 +0x29
compress/gzip.(*Writer).Close(0xc0014540b0, 0x42ef18, 0x2019938)
C:/go/src/compress/gzip/gzip.go:242 +0x75
github.com/docker/docker/layer.(*fileMetadataTransaction).TarSplitWriter.func1(0xc0006188c0, 0xc001264328)
e:/go/src/github.com/docker/docker/layer/filestore.go:123 +0x48
github.com/docker/docker/pkg/ioutils.(*writeCloserWrapper).Close(0xc001085980, 0x49a9c00, 0xc001085980)
e:/go/src/github.com/docker/docker/pkg/ioutils/writers.go:35 +0x32
github.com/docker/docker/layer.(*layerStore).applyTar(0xc000753860, 0xc000f7b7f0, 0x6e04710, 0xc0010851c0, 0xc000453f00, 0x40, 0xc000917860, 0x2010d20, 0xc0013ae590)
e:/go/src/github.com/docker/docker/layer/layer_store.go:257 +0x348
github.com/docker/docker/layer.(*layerStore).registerWithDescriptor(0xc000753860, 0x6e04710, 0xc0010851c0, 0xc000f7f220, 0x47, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
e:/go/src/github.com/docker/docker/layer/layer_store.go:334 +0x2bd
github.com/docker/docker/layer.(*layerStore).Register(0xc000753860, 0x6e04710, 0xc0010851c0, 0xc000f7f220, 0x47, 0x0, 0x2019820, 0xc0010851c0, 0x0)
e:/go/src/github.com/docker/docker/layer/layer_store.go:272 +0xab
github.com/docker/docker/daemon/images.(*ImageService).CommitImage(0xc000c2c870, 0x0, 0x0, 0x0, 0x0, 0xc000697540, 0xc00063da40, 0xc0010f8100, 0x40, 0x0, ...)
e:/go/src/github.com/docker/docker/daemon/images/image_commit.go:42 +0x29f
github.com/docker/docker/daemon/images.(*ImageService).CommitBuildStep(0xc000c2c870, 0x0, 0x0, 0x0, 0x0, 0xc000697540, 0xc00063da40, 0xc0010f8100, 0x40, 0x0, ...)
e:/go/src/github.com/docker/docker/daemon/images/image_commit.go:126 +0x102
github.com/docker/docker/builder/dockerfile.(*Builder).commitContainer(0xc000188370, 0xc000b19180, 0xc0010f8100, 0x40, 0xc00063da40, 0x0, 0x0)
e:/go/src/github.com/docker/docker/builder/dockerfile/internals.go:109 +0x12e
github.com/docker/docker/builder/dockerfile.(*Builder).commit(0xc000188370, 0xc000b19180, 0xc000ee7c20, 0x89, 0xc000ee7c20, 0x89)
e:/go/src/github.com/docker/docker/builder/dockerfile/internals.go:93 +0x14f
github.com/docker/docker/builder/dockerfile.dispatchEnv(0xc000b19180, 0xc0011a99e8, 0xc000188370, 0x2024e60, 0xc0001b03c0, 0xc00121b0c0, 0xc000996fc0, 0xc00121b0c0, 0xc000cc2d90)
e:/go/src/github.com/docker/docker/builder/dockerfile/dispatchers.go:62 +0x295
github.com/docker/docker/builder/dockerfile.dispatch(0xc000b19180, 0xc0011a99e8, 0xc000188370, 0x2024e60, 0xc0001b03c0, 0xc00121b0c0, 0x2012820, 0xc000996fc0, 0x0, 0x0)
e:/go/src/github.com/docker/docker/builder/dockerfile/evaluator.go:69 +0xa5a
github.com/docker/docker/builder/dockerfile.(*Builder).dispatchDockerfileWithCancellation(0xc000188370, 0xc0011ea6e0, 0x3, 0x4, 0x0, 0x0, 0x0, 0x5c, 0x2024e60, 0xc0001b03c0, ...)
e:/go/src/github.com/docker/docker/builder/dockerfile/builder.go:347 +0x505
github.com/docker/docker/builder/dockerfile.(*Builder).build(0xc000188370, 0x2024e60, 0xc0001b03c0, 0xc0011d7620, 0x0, 0x0, 0x0)
e:/go/src/github.com/docker/docker/builder/dockerfile/builder.go:265 +0x34c
github.com/docker/docker/builder/dockerfile.(*BuildManager).Build(0xc0009969c0, 0x2033760, 0xc000996d40, 0x20192a0, 0xc000ce8030, 0x2010da0, 0xc0001ac1e0, 0x2011540, 0xc0001ac200, 0x2011540, ...)
e:/go/src/github.com/docker/docker/builder/dockerfile/builder.go:120 +0x38d
github.com/docker/docker/api/server/backend/build.(*Backend).Build(0xc000a2b4d0, 0x2033760, 0xc0007d2b80, 0x20192a0, 0xc000ce8030, 0x2010da0, 0xc0001ac1e0, 0x2011540, 0xc0001ac200, 0x2011540, ...)
e:/go/src/github.com/docker/docker/api/server/backend/build/backend.go:60 +0x520
github.com/docker/docker/api/server/router/build.(*buildRouter).postBuild(0xc00008fbc0, 0x2033760, 0xc0007d2b80, 0x20318e0, 0xc000cd0000, 0xc000b40000, 0xc0001b01b0, 0x0, 0x0)
e:/go/src/github.com/docker/docker/api/server/router/build/build_routes.go:267 +0x5c8
github.com/docker/docker/api/server/router/build.(*buildRouter).postBuild-fm(0x2033760, 0xc0007d2b80, 0x20318e0, 0xc000cd0000, 0xc000b40000, 0xc0001b01b0, 0x1d9441d, 0xc000cc36a8)
e:/go/src/github.com/docker/docker/api/server/router/build/build.go:34 +0x70
github.com/docker/docker/api/server/router.cancellableHandler.func1(0x2033820, 0xc00091fd70, 0x20318e0, 0xc000cd0000, 0xc000b40000, 0xc0001b01b0, 0x0, 0x0)
e:/go/src/github.com/docker/docker/api/server/router/local.go:92 +0xd5
github.com/docker/docker/api/server/middleware.ExperimentalMiddleware.WrapHandler.func1(0x2033820, 0xc00091fd70, 0x20318e0, 0xc000cd0000, 0xc000b40000, 0xc0001b01b0, 0x2033820, 0xc00091fd70)
e:/go/src/github.com/docker/docker/api/server/middleware/experimental.go:26 +0xdf
github.com/docker/docker/api/server/middleware.VersionMiddleware.WrapHandler.func1(0x2033820, 0xc0001b0210, 0x20318e0, 0xc000cd0000, 0xc000b40000, 0xc0001b01b0, 0x18, 0xc000cc3900)
e:/go/src/github.com/docker/docker/api/server/middleware/version.go:62 +0x408
github.com/docker/docker/pkg/authorization.(*Middleware).WrapHandler.func1(0x2033820, 0xc0001b0210, 0x20318e0, 0xc000cd0000, 0xc000b40000, 0xc0001b01b0, 0x20022b8, 0x756ea185bfc640)
e:/go/src/github.com/docker/docker/pkg/authorization/middleware.go:59 +0x7c9
github.com/docker/docker/api/server/middleware.DebugRequestMiddleware.func1(0x2033820, 0xc0001b0210, 0x20318e0, 0xc000cd0000, 0xc000b40000, 0xc0001b01b0, 0x2033820, 0xc0001b0210)
e:/go/src/github.com/docker/docker/api/server/middleware/debug.go:25 +0x8b1
github.com/docker/docker/api/server.(*Server).makeHTTPHandler.func1(0x20318e0, 0xc000cd0000, 0xc000b40000)
e:/go/src/github.com/docker/docker/api/server/server.go:140 +0x19f
net/http.HandlerFunc.ServeHTTP(0xc00013abc0, 0x20318e0, 0xc000cd0000, 0xc000b40000)
C:/go/src/net/http/server.go:1964 +0x4b
github.com/docker/docker/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc000752460, 0x20318e0, 0xc000cd0000, 0xc000b40000)
e:/go/src/github.com/docker/docker/vendor/github.com/gorilla/mux/mux.go:103 +0x231
github.com/docker/docker/api/server.(*routerSwapper).ServeHTTP(0xc000714720, 0x20318e0, 0xc000cd0000, 0xc000b40000)
e:/go/src/github.com/docker/docker/api/server/router_swapper.go:29 +0x77
net/http.serverHandler.ServeHTTP(0xc000667d40, 0x20318e0, 0xc000cd0000, 0xc000b40000)
C:/go/src/net/http/server.go:2741 +0xb2
net/http.(*conn).serve(0xc000b74000, 0x2033760, 0xc000914040)
C:/go/src/net/http/server.go:1847 +0x64d
created by net/http.(*Server).Serve
C:/go/src/net/http/server.go:2851 +0x2fc
I don't understand this code well enough to figure out quite what's going on here, there's several levels of indirection involved. I could probably get there if I had more time, but I'm just opening the issue in the hope someone else can figure this out. Perhaps @vbatts, @tonistiigi, @thaJeztah @dnephin, @tiborvass or @duglin can come up with a solution. To repro, all you need to do is:
- In the appropriate graphdrivers ApplyDiff function,
return 0, fmt.Errorf("some error")at the top. Obviously I hit this in a far more subtle way in an LCOW driver change, but it applies to any graphdriver. - Perform a docker build at a step that commits a layer
By simply commenting out the tsw.Close() call, the CLI will reliably see something like the following (again, ignore the specific error, but just to demonstrate the hack works), and the daemon won't panic. Clearly, this isn't the right fix though.
Step 4/13 : ENV GOPATH=/go PATH=$PATH:/go/bin SRC=/go/src/github.com/Microsoft/opengcs
---> Running in b057a7794d74
path already in use: lost+found
The commented out bit to hack around this is:
if ls.useTarSplit {
tsw, err := tx.TarSplitWriter(true)
if err != nil {
return err
}
metaPacker := storage.NewJSONPacker(tsw)
---> THIS //defer tsw.Close()
// we're passing nil here for the file putter, because the ApplyDiff will
// handle the extraction of the archive
rdr, err = asm.NewInputTarStream(tr, metaPacker, nil)
if err != nil {
return err
}
}