Add ability to handle index acknowledgment with splunk log driver

Previously there was no way for the splunk log driver to work if index
acknowledgment was set on the HEC, and it would in fact fail silently.
This will now allow users to specify if index acknowledgment is set and
will work with that setting.

Signed-off-by: Devon Estes <devon.c.estes@gmail.com>

daemon/logger/splunk/splunk.go

@@ -24,6 +24,7 @@ import (
 	"github.com/docker/docker/daemon/logger/loggerutils"
 	"github.com/docker/docker/pkg/pools"
 	"github.com/docker/docker/pkg/urlutil"
+	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
 )
 
@@ -41,6 +42,7 @@ const (
 	splunkVerifyConnectionKey     = "splunk-verify-connection"
 	splunkGzipCompressionKey      = "splunk-gzip"
 	splunkGzipCompressionLevelKey = "splunk-gzip-level"
+	splunkIndexAcknowledgment     = "splunk-index-acknowledgment"
 	envKey                        = "env"
 	envRegexKey                   = "env-regex"
 	labelsKey                     = "labels"
@@ -91,6 +93,7 @@ type splunkLogger struct {
 	postMessagesFrequency time.Duration
 	postMessagesBatchSize int
 	bufferMaximum         int
+	indexAck              bool
 
 	// For synchronization between background worker and logger.
 	// We use channel to send messages to worker go routine.
@@ -217,6 +220,14 @@ func New(info logger.Info) (logger.Logger, error) {
 		}
 	}
 
+	indexAck := false
+	if indexAckStr, ok := info.Config[splunkIndexAcknowledgment]; ok {
+		indexAck, err = strconv.ParseBool(indexAckStr)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	transport := &http.Transport{
 		TLSClientConfig: tlsConfig,
 		Proxy:           http.ProxyFromEnvironment,
@@ -269,6 +280,7 @@ func New(info logger.Info) (logger.Logger, error) {
 		postMessagesFrequency: postMessagesFrequency,
 		postMessagesBatchSize: postMessagesBatchSize,
 		bufferMaximum:         bufferMaximum,
+		indexAck:              indexAck,
 	}
 
 	// By default we verify connection, but we allow use to skip that
@@ -505,6 +517,14 @@ func (l *splunkLogger) tryPostMessages(ctx context.Context, messages []*splunkMe
 	if l.gzipCompression {
 		req.Header.Set("Content-Encoding", "gzip")
 	}
+	// Set the correct header if index acknowledgment is enabled
+	if l.indexAck {
+		requestChannel, err := uuid.NewRandom()
+		if err != nil {
+			return err
+		}
+		req.Header.Set("X-Splunk-Request-Channel", requestChannel.String())
+	}
 	resp, err := l.client.Do(req)
 	if err != nil {
 		return err
@@ -563,6 +583,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 		case splunkVerifyConnectionKey:
 		case splunkGzipCompressionKey:
 		case splunkGzipCompressionLevelKey:
+		case splunkIndexAcknowledgment:
 		case envKey:
 		case envRegexKey:
 		case labelsKey: