228 lines
6.5 KiB
Go
228 lines
6.5 KiB
Go
package loggerutils // import "github.com/docker/docker/daemon/logger/loggerutils"
|
|
|
|
import (
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"runtime"
|
|
)
|
|
|
|
type fileConvertFn func(dst io.WriteSeeker, src io.ReadSeeker) error
|
|
|
|
type stfID uint64
|
|
|
|
// sharedTempFileConverter converts files using a user-supplied function and
|
|
// writes the results to temporary files which are automatically cleaned up on
|
|
// close. If another request is made to convert the same file, the conversion
|
|
// result and temporary file are reused if they have not yet been cleaned up.
|
|
//
|
|
// A file is considered the same as another file using the os.SameFile function,
|
|
// which compares file identity (e.g. device and inode numbers on Linux) and is
|
|
// robust to file renames. Input files are assumed to be immutable; no attempt
|
|
// is made to ascertain whether the file contents have changed between requests.
|
|
//
|
|
// One file descriptor is used per source file, irrespective of the number of
|
|
// concurrent readers of the converted contents.
|
|
type sharedTempFileConverter struct {
|
|
// The directory where temporary converted files are to be written to.
|
|
// If set to the empty string, the default directory for temporary files
|
|
// is used.
|
|
TempDir string
|
|
|
|
conv fileConvertFn
|
|
st chan stfcState
|
|
}
|
|
|
|
type stfcState struct {
|
|
fl map[stfID]sharedTempFile
|
|
nextID stfID
|
|
}
|
|
|
|
type sharedTempFile struct {
|
|
src os.FileInfo // Info about the source file for path-independent identification with os.SameFile.
|
|
fd *os.File
|
|
size int64
|
|
ref int // Reference count of open readers on the temporary file.
|
|
wait []chan<- stfConvertResult // Wait list for the conversion to complete.
|
|
}
|
|
|
|
type stfConvertResult struct {
|
|
fr *sharedFileReader
|
|
err error
|
|
}
|
|
|
|
func newSharedTempFileConverter(conv fileConvertFn) *sharedTempFileConverter {
|
|
st := make(chan stfcState, 1)
|
|
st <- stfcState{fl: make(map[stfID]sharedTempFile)}
|
|
return &sharedTempFileConverter{conv: conv, st: st}
|
|
}
|
|
|
|
// Do returns a reader for the contents of f as converted by the c.C function.
|
|
// It is the caller's responsibility to close the returned reader.
|
|
//
|
|
// This function is safe for concurrent use by multiple goroutines.
|
|
func (c *sharedTempFileConverter) Do(f *os.File) (*sharedFileReader, error) {
|
|
stat, err := f.Stat()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
st := <-c.st
|
|
for id, tf := range st.fl {
|
|
// os.SameFile can have false positives if one of the files was
|
|
// deleted before the other file was created -- such as during
|
|
// log rotations... https://github.com/golang/go/issues/36895
|
|
// Weed out those false positives by also comparing the files'
|
|
// ModTime, which conveniently also handles the case of true
|
|
// positives where the file has also been modified since it was
|
|
// first converted.
|
|
if os.SameFile(tf.src, stat) && tf.src.ModTime() == stat.ModTime() {
|
|
return c.openExisting(st, id, tf)
|
|
}
|
|
}
|
|
return c.openNew(st, f, stat)
|
|
}
|
|
|
|
func (c *sharedTempFileConverter) openNew(st stfcState, f *os.File, stat os.FileInfo) (*sharedFileReader, error) {
|
|
// Record that we are starting to convert this file so that any other
|
|
// requests for the same source file while the conversion is in progress
|
|
// can join.
|
|
id := st.nextID
|
|
st.nextID++
|
|
st.fl[id] = sharedTempFile{src: stat}
|
|
c.st <- st
|
|
|
|
dst, size, convErr := c.convert(f)
|
|
|
|
st = <-c.st
|
|
flid := st.fl[id]
|
|
|
|
if convErr != nil {
|
|
// Conversion failed. Delete it from the state so that future
|
|
// requests to convert the same file can try again fresh.
|
|
delete(st.fl, id)
|
|
c.st <- st
|
|
for _, w := range flid.wait {
|
|
w <- stfConvertResult{err: convErr}
|
|
}
|
|
return nil, convErr
|
|
}
|
|
|
|
flid.fd = dst
|
|
flid.size = size
|
|
flid.ref = len(flid.wait) + 1
|
|
for _, w := range flid.wait {
|
|
// Each waiter needs its own reader with an independent read pointer.
|
|
w <- stfConvertResult{fr: flid.Reader(c, id)}
|
|
}
|
|
flid.wait = nil
|
|
st.fl[id] = flid
|
|
c.st <- st
|
|
return flid.Reader(c, id), nil
|
|
}
|
|
|
|
func (c *sharedTempFileConverter) openExisting(st stfcState, id stfID, v sharedTempFile) (*sharedFileReader, error) {
|
|
if v.fd != nil {
|
|
// Already converted.
|
|
v.ref++
|
|
st.fl[id] = v
|
|
c.st <- st
|
|
return v.Reader(c, id), nil
|
|
}
|
|
// The file has not finished being converted.
|
|
// Add ourselves to the wait list. "Don't call us; we'll call you."
|
|
wait := make(chan stfConvertResult, 1)
|
|
v.wait = append(v.wait, wait)
|
|
st.fl[id] = v
|
|
c.st <- st
|
|
|
|
res := <-wait
|
|
return res.fr, res.err
|
|
|
|
}
|
|
|
|
func (c *sharedTempFileConverter) convert(f *os.File) (converted *os.File, size int64, err error) {
|
|
dst, err := os.CreateTemp(c.TempDir, "dockerdtemp.*")
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
defer func() {
|
|
_ = dst.Close()
|
|
// Delete the temporary file immediately so that final cleanup
|
|
// of the file on disk is deferred to the OS once we close all
|
|
// our file descriptors (or the process dies). Assuming no early
|
|
// returns due to errors, the file will be open by this process
|
|
// with a read-only descriptor at this point. As we don't care
|
|
// about being able to reuse the file name -- it's randomly
|
|
// generated and unique -- we can safely use os.Remove on
|
|
// Windows.
|
|
_ = os.Remove(dst.Name())
|
|
}()
|
|
err = c.conv(dst, f)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
// Close the exclusive read-write file descriptor, catching any delayed
|
|
// write errors (and on Windows, releasing the share-locks on the file)
|
|
if err := dst.Close(); err != nil {
|
|
_ = os.Remove(dst.Name())
|
|
return nil, 0, err
|
|
}
|
|
// Open the file again read-only (without locking the file against
|
|
// deletion on Windows).
|
|
converted, err = open(dst.Name())
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
// The position of the file's read pointer doesn't matter as all readers
|
|
// will be accessing the file through its io.ReaderAt interface.
|
|
size, err = converted.Seek(0, io.SeekEnd)
|
|
if err != nil {
|
|
_ = converted.Close()
|
|
return nil, 0, err
|
|
}
|
|
return converted, size, nil
|
|
}
|
|
|
|
type sharedFileReader struct {
|
|
*io.SectionReader
|
|
|
|
c *sharedTempFileConverter
|
|
id stfID
|
|
closed bool
|
|
}
|
|
|
|
func (stf sharedTempFile) Reader(c *sharedTempFileConverter, id stfID) *sharedFileReader {
|
|
rdr := &sharedFileReader{SectionReader: io.NewSectionReader(stf.fd, 0, stf.size), c: c, id: id}
|
|
runtime.SetFinalizer(rdr, (*sharedFileReader).Close)
|
|
return rdr
|
|
}
|
|
|
|
func (r *sharedFileReader) Close() error {
|
|
if r.closed {
|
|
return fs.ErrClosed
|
|
}
|
|
|
|
st := <-r.c.st
|
|
flid, ok := st.fl[r.id]
|
|
if !ok {
|
|
panic("invariant violation: temp file state missing from map")
|
|
}
|
|
flid.ref--
|
|
lastRef := flid.ref <= 0
|
|
if lastRef {
|
|
delete(st.fl, r.id)
|
|
} else {
|
|
st.fl[r.id] = flid
|
|
}
|
|
r.closed = true
|
|
r.c.st <- st
|
|
|
|
if lastRef {
|
|
return flid.fd.Close()
|
|
}
|
|
runtime.SetFinalizer(r, nil)
|
|
return nil
|
|
}
|