From 4c62b1263648833122828cf8cc8259f5b7e32da5 Mon Sep 17 00:00:00 2001 From: John Howard Date: Wed, 12 Oct 2016 15:29:47 -0700 Subject: [PATCH] Spew debugging Signed-off-by: John Howard --- daemon/daemon.go | 8 ++++---- daemon/debugtrap.go | 33 +++++++++++++++++++++++++++++++++ daemon/debugtrap_unix.go | 15 ++++++++++----- daemon/debugtrap_unsupported.go | 2 +- daemon/debugtrap_windows.go | 12 +++++++++--- pkg/signal/trap.go | 3 ++- 6 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 daemon/debugtrap.go diff --git a/daemon/daemon.go b/daemon/daemon.go index 97c81a5e9f..c66d3de363 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -476,10 +476,6 @@ func NewDaemon(config *Config, registryService registry.Service, containerdRemot return nil, err } - // set up SIGUSR1 handler on Unix-like systems, or a Win32 global event - // on Windows to dump Go routine stacks - setupDumpStackTrap(config.Root) - uidMaps, gidMaps, err := setupRemappedRoot(config) if err != nil { return nil, err @@ -699,6 +695,10 @@ func NewDaemon(config *Config, registryService registry.Service, containerdRemot engineCpus.Set(float64(info.NCPU)) engineMemory.Set(float64(info.MemTotal)) + // set up SIGUSR1 handler on Unix-like systems, or a Win32 global event + // on Windows to dump Go routine stacks + d.setupDumpStackTrap(config.Root) + return d, nil } diff --git a/daemon/debugtrap.go b/daemon/debugtrap.go new file mode 100644 index 0000000000..0a7c5b594a --- /dev/null +++ b/daemon/debugtrap.go @@ -0,0 +1,33 @@ +package daemon + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/pkg/errors" +) + +const dataStructuresLogNameTemplate = "daemon-data-%s.log" + +// dumpDaemon appends the daemon datastructures into file in dir and returns full path +// to that file. +func (d *Daemon) dumpDaemon(dir string) (string, error) { + // Ensure we recover from a panic as we are doing this without any locking + defer func() { + recover() + }() + + path := filepath.Join(dir, fmt.Sprintf(dataStructuresLogNameTemplate, strings.Replace(time.Now().Format(time.RFC3339), ":", "", -1))) + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return "", errors.Wrap(err, "failed to open file to write the daemon datastructure dump") + } + defer f.Close() + spew.Fdump(f, d) // Does not return an error + f.Sync() + return path, nil +} diff --git a/daemon/debugtrap_unix.go b/daemon/debugtrap_unix.go index 1b22fdf1c9..d650eb7f8c 100644 --- a/daemon/debugtrap_unix.go +++ b/daemon/debugtrap_unix.go @@ -7,12 +7,11 @@ import ( "os/signal" "syscall" - stackdump "github.com/docker/docker/pkg/signal" - "github.com/Sirupsen/logrus" + stackdump "github.com/docker/docker/pkg/signal" ) -func setupDumpStackTrap(root string) { +func (d *Daemon) setupDumpStackTrap(root string) { c := make(chan os.Signal, 1) signal.Notify(c, syscall.SIGUSR1) go func() { @@ -20,9 +19,15 @@ func setupDumpStackTrap(root string) { path, err := stackdump.DumpStacks(root) if err != nil { logrus.WithError(err).Error("failed to write goroutines dump") - continue + } else { + logrus.Infof("goroutine stacks written to %s", path) + } + path, err = d.dumpDaemon(root) + if err != nil { + logrus.WithError(err).Error("failed to write daemon datastructure dump") + } else { + logrus.Infof("daemon datastructure dump written to %s", path) } - logrus.Infof("goroutine stacks written to %s", path) } }() } diff --git a/daemon/debugtrap_unsupported.go b/daemon/debugtrap_unsupported.go index fba4a95e06..f5b9170907 100644 --- a/daemon/debugtrap_unsupported.go +++ b/daemon/debugtrap_unsupported.go @@ -2,6 +2,6 @@ package daemon -func setupDumpStackTrap(_ string) { +func (d *Daemon) setupDumpStackTrap(_ string) { return } diff --git a/daemon/debugtrap_windows.go b/daemon/debugtrap_windows.go index 678672253c..fb20c9d2c5 100644 --- a/daemon/debugtrap_windows.go +++ b/daemon/debugtrap_windows.go @@ -12,7 +12,7 @@ import ( "github.com/docker/docker/pkg/system" ) -func setupDumpStackTrap(root string) { +func (d *Daemon) setupDumpStackTrap(root string) { // Windows does not support signals like *nix systems. So instead of // trapping on SIGUSR1 to dump stacks, we wait on a Win32 event to be // signaled. ACL'd to builtin administrators and local system @@ -38,9 +38,15 @@ func setupDumpStackTrap(root string) { path, err := signal.DumpStacks(root) if err != nil { logrus.WithError(err).Error("failed to write goroutines dump") - continue + } else { + logrus.Infof("goroutine stacks written to %s", path) + } + path, err = d.dumpDaemon(root) + if err != nil { + logrus.WithError(err).Error("failed to write daemon datastructure dump") + } else { + logrus.Infof("daemon datastructure dump written to %s", path) } - logrus.Infof("goroutine stacks written to %s", path) } }() } diff --git a/pkg/signal/trap.go b/pkg/signal/trap.go index 120c34aab2..44c578aef5 100644 --- a/pkg/signal/trap.go +++ b/pkg/signal/trap.go @@ -6,6 +6,7 @@ import ( gosignal "os/signal" "path/filepath" "runtime" + "strings" "sync/atomic" "syscall" "time" @@ -82,7 +83,7 @@ func DumpStacks(dir string) (string, error) { bufferLen *= 2 } buf = buf[:stackSize] - path := filepath.Join(dir, fmt.Sprintf(stacksLogNameTemplate, time.Now().Format(time.RFC3339))) + path := filepath.Join(dir, fmt.Sprintf(stacksLogNameTemplate, strings.Replace(time.Now().Format(time.RFC3339), ":", "", -1))) f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return "", errors.Wrap(err, "failed to open file to write the goroutine stacks")