Default 'json-file' logging driver and none logging driver

Signed-off-by: Alexander Morozov <lk4d4@docker.com>

daemon/config.go

@@ -7,6 +7,7 @@ import (
 	"github.com/docker/docker/opts"
 	flag "github.com/docker/docker/pkg/mflag"
 	"github.com/docker/docker/pkg/ulimit"
+	"github.com/docker/docker/runconfig"
 )
 
 const (
@@ -47,6 +48,7 @@ type Config struct {
 	TrustKeyPath                string
 	Labels                      []string
 	Ulimits                     map[string]*ulimit.Ulimit
+	LogConfig                   runconfig.LogConfig
 }
 
 // InstallFlags adds command-line options to the top-level flag parser for
@@ -81,6 +83,7 @@ func (config *Config) InstallFlags() {
 	opts.LabelListVar(&config.Labels, []string{"-label"}, "Set key=value labels to the daemon")
 	config.Ulimits = make(map[string]*ulimit.Ulimit)
 	opts.UlimitMapVar(config.Ulimits, []string{"-default-ulimit"}, "Set default ulimits for containers")
+	flag.StringVar(&config.LogConfig.Type, []string{"-log-driver"}, "json-file", "Containers logging driver(json-file/none)")
 }
 
 func getDefaultNetworkMtu() int {

daemon/container.go

@@ -21,6 +21,8 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/execdriver"
+	"github.com/docker/docker/daemon/logger"
+	"github.com/docker/docker/daemon/logger/jsonfilelog"
 	"github.com/docker/docker/engine"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/links"
@@ -98,9 +100,11 @@ type Container struct {
 	VolumesRW  map[string]bool
 	hostConfig *runconfig.HostConfig
 
-	activeLinks        map[string]*links.Link
-	monitor            *containerMonitor
-	execCommands       *execStore
+	activeLinks  map[string]*links.Link
+	monitor      *containerMonitor
+	execCommands *execStore
+	// logDriver for closing
+	logDriver          logger.Logger
 	AppliedVolumesFrom map[string]struct{}
 }
 
@@ -1355,21 +1359,36 @@ func (container *Container) setupWorkingDirectory() error {
 	return nil
 }
 
-func (container *Container) startLoggingToDisk() error {
-	// Setup logging of stdout and stderr to disk
-	logPath, err := container.logPath("json")
-	if err != nil {
-		return err
+func (container *Container) startLogging() error {
+	cfg := container.hostConfig.LogConfig
+	if cfg.Type == "" {
+		cfg = container.daemon.defaultLogConfig
 	}
-	container.LogPath = logPath
+	var l logger.Logger
+	switch cfg.Type {
+	case "json-file":
+		pth, err := container.logPath("json")
+		if err != nil {
+			return err
+		}
 
-	if err := container.daemon.LogToDisk(container.stdout, container.LogPath, "stdout"); err != nil {
-		return err
+		dl, err := jsonfilelog.New(pth)
+		if err != nil {
+			return err
+		}
+		l = dl
+	case "none":
+		return nil
+	default:
+		return fmt.Errorf("Unknown logging driver: %s", cfg.Type)
 	}
 
-	if err := container.daemon.LogToDisk(container.stderr, container.LogPath, "stderr"); err != nil {
+	if copier, err := logger.NewCopier(container.ID, map[string]io.Reader{"stdout": container.StdoutPipe(), "stderr": container.StderrPipe()}, l); err != nil {
 		return err
+	} else {
+		copier.Run()
 	}
+	container.logDriver = l
 
 	return nil
 }

daemon/daemon.go

@@ -89,23 +89,24 @@ func (c *contStore) List() []*Container {
 }
 
 type Daemon struct {
-	ID             string
-	repository     string
-	sysInitPath    string
-	containers     *contStore
-	execCommands   *execStore
-	graph          *graph.Graph
-	repositories   *graph.TagStore
-	idIndex        *truncindex.TruncIndex
-	sysInfo        *sysinfo.SysInfo
-	volumes        *volumes.Repository
-	eng            *engine.Engine
-	config         *Config
-	containerGraph *graphdb.Database
-	driver         graphdriver.Driver
-	execDriver     execdriver.Driver
-	trustStore     *trust.TrustStore
-	statsCollector *statsCollector
+	ID               string
+	repository       string
+	sysInitPath      string
+	containers       *contStore
+	execCommands     *execStore
+	graph            *graph.Graph
+	repositories     *graph.TagStore
+	idIndex          *truncindex.TruncIndex
+	sysInfo          *sysinfo.SysInfo
+	volumes          *volumes.Repository
+	eng              *engine.Engine
+	config           *Config
+	containerGraph   *graphdb.Database
+	driver           graphdriver.Driver
+	execDriver       execdriver.Driver
+	trustStore       *trust.TrustStore
+	statsCollector   *statsCollector
+	defaultLogConfig runconfig.LogConfig
 }
 
 // Install installs daemon capabilities to eng.
@@ -984,23 +985,24 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error)
 	}
 
 	daemon := &Daemon{
-		ID:             trustKey.PublicKey().KeyID(),
-		repository:     daemonRepo,
-		containers:     &contStore{s: make(map[string]*Container)},
-		execCommands:   newExecStore(),
-		graph:          g,
-		repositories:   repositories,
-		idIndex:        truncindex.NewTruncIndex([]string{}),
-		sysInfo:        sysInfo,
-		volumes:        volumes,
-		config:         config,
-		containerGraph: graph,
-		driver:         driver,
-		sysInitPath:    sysInitPath,
-		execDriver:     ed,
-		eng:            eng,
-		trustStore:     t,
-		statsCollector: newStatsCollector(1 * time.Second),
+		ID:               trustKey.PublicKey().KeyID(),
+		repository:       daemonRepo,
+		containers:       &contStore{s: make(map[string]*Container)},
+		execCommands:     newExecStore(),
+		graph:            g,
+		repositories:     repositories,
+		idIndex:          truncindex.NewTruncIndex([]string{}),
+		sysInfo:          sysInfo,
+		volumes:          volumes,
+		config:           config,
+		containerGraph:   graph,
+		driver:           driver,
+		sysInitPath:      sysInitPath,
+		execDriver:       ed,
+		eng:              eng,
+		trustStore:       t,
+		statsCollector:   newStatsCollector(1 * time.Second),
+		defaultLogConfig: config.LogConfig,
 	}
 
 	// Setup shutdown handlers

daemon/inspect.go

@@ -62,6 +62,14 @@ func (daemon *Daemon) ContainerInspect(job *engine.Job) engine.Status {
 			container.hostConfig.Links = append(container.hostConfig.Links, fmt.Sprintf("%s:%s", child.Name, linkAlias))
 		}
 	}
+	// we need this trick to preserve empty log driver, so
+	// container will use daemon defaults even if daemon change them
+	if container.hostConfig.LogConfig.Type == "" {
+		container.hostConfig.LogConfig = daemon.defaultLogConfig
+		defer func() {
+			container.hostConfig.LogConfig = runconfig.LogConfig{}
+		}()
+	}
 
 	out.SetJson("HostConfig", container.hostConfig)
 

daemon/logger/jsonfilelog/jsonfilelog.go

@@ -0,0 +1,49 @@
+package jsonfilelog
+
+import (
+	"bytes"
+	"os"
+
+	"github.com/docker/docker/daemon/logger"
+	"github.com/docker/docker/pkg/jsonlog"
+)
+
+// JSONFileLogger is Logger implementation for default docker logging:
+// JSON objects to file
+type JSONFileLogger struct {
+	buf *bytes.Buffer
+	f   *os.File // store for closing
+}
+
+// New creates new JSONFileLogger which writes to filename
+func New(filename string) (logger.Logger, error) {
+	log, err := os.OpenFile(filename, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600)
+	if err != nil {
+		return nil, err
+	}
+	return &JSONFileLogger{
+		f:   log,
+		buf: bytes.NewBuffer(nil),
+	}, nil
+}
+
+// Log converts logger.Message to jsonlog.JSONLog and serializes it to file
+func (l *JSONFileLogger) Log(msg *logger.Message) error {
+	err := (&jsonlog.JSONLog{Log: string(msg.Line) + "\n", Stream: msg.Source, Created: msg.Timestamp}).MarshalJSONBuf(l.buf)
+	if err != nil {
+		return err
+	}
+	l.buf.WriteByte('\n')
+	_, err = l.buf.WriteTo(l.f)
+	return err
+}
+
+// Close closes underlying file
+func (l *JSONFileLogger) Close() error {
+	return l.f.Close()
+}
+
+// Name returns name of this logger
+func (l *JSONFileLogger) Name() string {
+	return "JSONFile"
+}

daemon/logger/jsonfilelog/jsonfilelog_test.go

@@ -0,0 +1,78 @@
+package jsonfilelog
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/daemon/logger"
+	"github.com/docker/docker/pkg/jsonlog"
+)
+
+func TestJSONFileLogger(t *testing.T) {
+	tmp, err := ioutil.TempDir("", "docker-logger-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	filename := filepath.Join(tmp, "container.log")
+	l, err := New(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657"
+	if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line1"), Source: "src1"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line2"), Source: "src2"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line3"), Source: "src3"}); err != nil {
+		t.Fatal(err)
+	}
+	res, err := ioutil.ReadFile(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := `{"log":"line1\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
+{"log":"line2\n","stream":"src2","time":"0001-01-01T00:00:00Z"}
+{"log":"line3\n","stream":"src3","time":"0001-01-01T00:00:00Z"}
+`
+
+	if string(res) != expected {
+		t.Fatalf("Wrong log content: %q, expected %q", res, expected)
+	}
+}
+
+func BenchmarkJSONFileLogger(b *testing.B) {
+	tmp, err := ioutil.TempDir("", "docker-logger-")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	filename := filepath.Join(tmp, "container.log")
+	l, err := New(filename)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer l.Close()
+	cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657"
+	testLine := "Line that thinks that it is log line from docker\n"
+	msg := &logger.Message{ContainerID: cid, Line: []byte(testLine), Source: "stderr", Timestamp: time.Now().UTC()}
+	jsonlog, err := (&jsonlog.JSONLog{Log: string(msg.Line) + "\n", Stream: msg.Source, Created: msg.Timestamp}).MarshalJSON()
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.SetBytes(int64(len(jsonlog)+1) * 30)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for j := 0; j < 30; j++ {
+			if err := l.Log(msg); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}

daemon/monitor.go

@@ -123,7 +123,7 @@ func (m *containerMonitor) Start() error {
 	for {
 		m.container.RestartCount++
 
-		if err := m.container.startLoggingToDisk(); err != nil {
+		if err := m.container.startLogging(); err != nil {
 			m.resetContainer(false)
 
 			return err
@@ -302,6 +302,11 @@ func (m *containerMonitor) resetContainer(lock bool) {
 		container.stdin, container.stdinPipe = io.Pipe()
 	}
 
+	if container.logDriver != nil {
+		container.logDriver.Close()
+		container.logDriver = nil
+	}
+
 	c := container.command.ProcessConfig.Cmd
 
 	container.command.ProcessConfig.Cmd = exec.Cmd{

docs/sources/reference/api/docker_remote_api_v1.18.md

@@ -156,7 +156,8 @@ Create a container
                "RestartPolicy": { "Name": "", "MaximumRetryCount": 0 },
                "NetworkMode": "bridge",
                "Devices": [],
-               "Ulimits": [{}]
+               "Ulimits": [{}],
+               "LogConfig": { "Type": "json-file", Config: {} }
             }
         }
 
@@ -248,6 +249,9 @@ Json Parameters:
   -   **Ulimits** - A list of ulimits to be set in the container, specified as
         `{ "Name": <name>, "Soft": <soft limit>, "Hard": <hard limit> }`, for example:
         `Ulimits: { "Name": "nofile", "Soft": 1024, "Hard", 2048 }}`
+  -   **LogConfig** - Logging configuration to container, format
+        `{ "Type": "<driver_name>", "Config": {"key1": "val1"}}
+        Available types: `json-file`, `none`.
 
 Query Parameters:
 
@@ -340,6 +344,7 @@ Return low-level information on the container `id`
 				"MaximumRetryCount": 2,
 				"Name": "on-failure"
 			},
+           "LogConfig": { "Type": "json-file", Config: {} },
 			"SecurityOpt": null,
 			"VolumesFrom": null,
 			"Ulimits": [{}]

docs/sources/reference/commandline/cli.md

@@ -97,6 +97,7 @@ expect an integer, and they can only be specified once.
       --ipv6=false                           Enable IPv6 networking
       -l, --log-level="info"                 Set the logging level
       --label=[]                             Set key=value labels to the daemon
+      --log-driver="json-file"               Container's logging driver (json-file/none)
       --mtu=0                                Set the containers network MTU
       -p, --pidfile="/var/run/docker.pid"    Path to use for daemon PID file
       --registry-mirror=[]                   Preferred Docker registry mirror
@@ -792,6 +793,7 @@ Creates a new container.
       -i, --interactive=false    Keep STDIN open even if not attached
       --ipc=""                   IPC namespace to use
       --link=[]                  Add link to another container
+      --log-driver=""            Logging driver for container
       --lxc-conf=[]              Add custom lxc options
       -m, --memory=""            Memory limit
       --mac-address=""           Container MAC address (e.g. 92:d0:c6:0a:29:33)
@@ -1660,6 +1662,7 @@ removed before the image is removed.
       -i, --interactive=false    Keep STDIN open even if not attached
       --ipc=""                   IPC namespace to use
       --link=[]                  Add link to another container
+      --log-driver=""            Logging driver for container
       --lxc-conf=[]              Add custom lxc options
       -m, --memory=""            Memory limit
       --mac-address=""           Container MAC address (e.g. 92:d0:c6:0a:29:33)

docs/sources/reference/run.md

@@ -561,6 +561,18 @@ familiar with using LXC directly.
 > you can use `--lxc-conf` to set a container's IP address, but this will not be
 > reflected in the `/etc/hosts` file.
 
+## Logging drivers (--log-driver)
+
+You can specify a different logging driver for the container than for the daemon.
+
+### Log driver: none
+
+Disables any logging for the container.
+
+### Log driver: json-file
+
+Default logging driver for Docker. Writes JSON messages to file.
+
 ## Overriding Dockerfile image defaults
 
 When a developer builds an image from a [*Dockerfile*](/reference/builder)

integration/utils_test.go

@@ -191,6 +191,7 @@ func newTestEngine(t Fataler, autorestart bool, root string) *engine.Engine {
 		// otherwise NewDaemon will fail because of conflicting settings.
 		InterContainerCommunication: true,
 		TrustKeyPath:                filepath.Join(root, "key.json"),
+		LogConfig:                   runconfig.LogConfig{Type: "json-file"},
 	}
 	d, err := daemon.NewDaemon(cfg, eng)
 	if err != nil {

runconfig/hostconfig.go

@@ -99,6 +99,11 @@ type RestartPolicy struct {
 	MaximumRetryCount int
 }
 
+type LogConfig struct {
+	Type   string
+	Config map[string]string
+}
+
 type HostConfig struct {
 	Binds           []string
 	ContainerIDFile string
@@ -121,6 +126,7 @@ type HostConfig struct {
 	SecurityOpt     []string
 	ReadonlyRootfs  bool
 	Ulimits         []*ulimit.Ulimit
+	LogConfig       LogConfig
 }
 
 // This is used by the create command when you want to set both the
@@ -158,9 +164,8 @@ func ContainerHostConfigFromJob(job *engine.Job) *HostConfig {
 	job.GetenvJson("PortBindings", &hostConfig.PortBindings)
 	job.GetenvJson("Devices", &hostConfig.Devices)
 	job.GetenvJson("RestartPolicy", &hostConfig.RestartPolicy)
-
 	job.GetenvJson("Ulimits", &hostConfig.Ulimits)
-
+	job.GetenvJson("LogConfig", &hostConfig.LogConfig)
 	hostConfig.SecurityOpt = job.GetenvList("SecurityOpt")
 	if Binds := job.GetenvList("Binds"); Binds != nil {
 		hostConfig.Binds = Binds

runconfig/parse.go

@@ -68,6 +68,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe
 		flIpcMode         = cmd.String([]string{"-ipc"}, "", "IPC namespace to use")
 		flRestartPolicy   = cmd.String([]string{"-restart"}, "no", "Restart policy to apply when a container exits")
 		flReadonlyRootfs  = cmd.Bool([]string{"-read-only"}, false, "Mount the container's root filesystem as read only")
+		flLoggingDriver   = cmd.String([]string{"-log-driver"}, "", "Logging driver for container")
 	)
 
 	cmd.Var(&flAttach, []string{"a", "-attach"}, "Attach to STDIN, STDOUT or STDERR")
@@ -321,6 +322,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe
 		SecurityOpt:     flSecurityOpt.GetAll(),
 		ReadonlyRootfs:  *flReadonlyRootfs,
 		Ulimits:         flUlimits.GetList(),
+		LogConfig:       LogConfig{Type: *flLoggingDriver},
 	}
 
 	// When allocating stdin in attached mode, close stdin at client disconnect