2014-07-30 13:42:12 +00:00
|
|
|
package tarsum
|
2013-07-17 19:13:22 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
|
|
|
"compress/gzip"
|
|
|
|
"crypto/sha256"
|
|
|
|
"encoding/hex"
|
|
|
|
"hash"
|
|
|
|
"io"
|
2013-12-27 00:02:15 +00:00
|
|
|
"strings"
|
2014-07-30 13:42:12 +00:00
|
|
|
|
|
|
|
"github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
|
2014-07-24 20:37:44 +00:00
|
|
|
|
2014-10-24 17:12:35 +00:00
|
|
|
log "github.com/Sirupsen/logrus"
|
2013-07-17 19:13:22 +00:00
|
|
|
)
|
|
|
|
|
2014-09-06 12:29:19 +00:00
|
|
|
const (
|
|
|
|
buf8K = 8 * 1024
|
|
|
|
buf16K = 16 * 1024
|
|
|
|
buf32K = 32 * 1024
|
|
|
|
)
|
|
|
|
|
2014-08-21 20:12:52 +00:00
|
|
|
// NewTarSum creates a new interface for calculating a fixed time checksum of a
|
|
|
|
// tar archive.
|
|
|
|
//
|
|
|
|
// This is used for calculating checksums of layers of an image, in some cases
|
|
|
|
// including the byte payload of the image's json metadata as well, and for
|
|
|
|
// calculating the checksums for buildcache.
|
2014-09-11 01:56:20 +00:00
|
|
|
func NewTarSum(r io.Reader, dc bool, v Version) (TarSum, error) {
|
2014-11-19 20:46:03 +00:00
|
|
|
return NewTarSumHash(r, dc, v, DefaultTHash)
|
2014-08-21 20:12:52 +00:00
|
|
|
}
|
|
|
|
|
2014-05-15 20:50:58 +00:00
|
|
|
// Create a new TarSum, providing a THash to use rather than the DefaultTHash
|
|
|
|
func NewTarSumHash(r io.Reader, dc bool, v Version, tHash THash) (TarSum, error) {
|
2014-10-30 20:47:31 +00:00
|
|
|
headerSelector, err := getTarHeaderSelector(v)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2014-05-15 20:50:58 +00:00
|
|
|
}
|
2014-11-19 20:46:03 +00:00
|
|
|
ts := &tarSum{Reader: r, DisableCompression: dc, tarSumVersion: v, headerSelector: headerSelector, tHash: tHash}
|
|
|
|
err = ts.initTarSum()
|
|
|
|
return ts, err
|
2014-05-15 20:50:58 +00:00
|
|
|
}
|
|
|
|
|
2014-09-11 01:56:20 +00:00
|
|
|
// TarSum is the generic interface for calculating fixed time
|
2014-08-21 20:12:52 +00:00
|
|
|
// checksums of a tar archive
|
2014-09-11 01:56:20 +00:00
|
|
|
type TarSum interface {
|
2014-08-21 20:12:52 +00:00
|
|
|
io.Reader
|
2014-09-04 20:13:50 +00:00
|
|
|
GetSums() FileInfoSums
|
2014-08-21 20:12:52 +00:00
|
|
|
Sum([]byte) string
|
|
|
|
Version() Version
|
2014-05-15 20:50:58 +00:00
|
|
|
Hash() THash
|
2014-08-21 20:12:52 +00:00
|
|
|
}
|
|
|
|
|
2014-09-11 01:56:20 +00:00
|
|
|
// tarSum struct is the structure for a Version0 checksum calculation
|
|
|
|
type tarSum struct {
|
2013-07-17 19:13:22 +00:00
|
|
|
io.Reader
|
2014-01-08 01:46:04 +00:00
|
|
|
tarR *tar.Reader
|
|
|
|
tarW *tar.Writer
|
2014-05-15 20:50:58 +00:00
|
|
|
writer writeCloseFlusher
|
2014-01-08 01:46:04 +00:00
|
|
|
bufTar *bytes.Buffer
|
2014-05-15 20:50:58 +00:00
|
|
|
bufWriter *bytes.Buffer
|
2014-09-06 12:29:19 +00:00
|
|
|
bufData []byte
|
2014-01-08 01:46:04 +00:00
|
|
|
h hash.Hash
|
2014-05-15 20:50:58 +00:00
|
|
|
tHash THash
|
2014-09-04 20:13:50 +00:00
|
|
|
sums FileInfoSums
|
|
|
|
fileCounter int64
|
2014-01-08 01:46:04 +00:00
|
|
|
currentFile string
|
|
|
|
finished bool
|
|
|
|
first bool
|
2014-10-30 20:47:31 +00:00
|
|
|
DisableCompression bool // false by default. When false, the output gzip compressed.
|
|
|
|
tarSumVersion Version // this field is not exported so it can not be mutated during use
|
|
|
|
headerSelector tarHeaderSelector // handles selecting and ordering headers for files in the archive
|
2014-01-08 01:46:04 +00:00
|
|
|
}
|
|
|
|
|
2014-05-15 20:50:58 +00:00
|
|
|
func (ts tarSum) Hash() THash {
|
|
|
|
return ts.tHash
|
|
|
|
}
|
|
|
|
|
2014-09-11 01:56:20 +00:00
|
|
|
func (ts tarSum) Version() Version {
|
2014-08-21 20:12:52 +00:00
|
|
|
return ts.tarSumVersion
|
2014-01-08 01:46:04 +00:00
|
|
|
}
|
|
|
|
|
2014-05-15 20:50:58 +00:00
|
|
|
// A hash.Hash type generator and its name
|
|
|
|
type THash interface {
|
|
|
|
Hash() hash.Hash
|
|
|
|
Name() string
|
|
|
|
}
|
|
|
|
|
|
|
|
// Convenience method for creating a THash
|
|
|
|
func NewTHash(name string, h func() hash.Hash) THash {
|
|
|
|
return simpleTHash{n: name, h: h}
|
|
|
|
}
|
|
|
|
|
|
|
|
// TarSum default is "sha256"
|
|
|
|
var DefaultTHash = NewTHash("sha256", sha256.New)
|
|
|
|
|
|
|
|
type simpleTHash struct {
|
|
|
|
n string
|
|
|
|
h func() hash.Hash
|
|
|
|
}
|
|
|
|
|
|
|
|
func (sth simpleTHash) Name() string { return sth.n }
|
|
|
|
func (sth simpleTHash) Hash() hash.Hash { return sth.h() }
|
|
|
|
|
2014-09-11 01:56:20 +00:00
|
|
|
func (ts *tarSum) encodeHeader(h *tar.Header) error {
|
2014-10-30 20:47:31 +00:00
|
|
|
for _, elem := range ts.headerSelector.selectHeaders(h) {
|
2013-07-17 19:13:22 +00:00
|
|
|
if _, err := ts.h.Write([]byte(elem[0] + elem[1])); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2014-05-15 20:50:58 +00:00
|
|
|
func (ts *tarSum) initTarSum() error {
|
|
|
|
ts.bufTar = bytes.NewBuffer([]byte{})
|
|
|
|
ts.bufWriter = bytes.NewBuffer([]byte{})
|
|
|
|
ts.tarR = tar.NewReader(ts.Reader)
|
|
|
|
ts.tarW = tar.NewWriter(ts.bufTar)
|
|
|
|
if !ts.DisableCompression {
|
|
|
|
ts.writer = gzip.NewWriter(ts.bufWriter)
|
|
|
|
} else {
|
|
|
|
ts.writer = &nopCloseFlusher{Writer: ts.bufWriter}
|
|
|
|
}
|
|
|
|
if ts.tHash == nil {
|
|
|
|
ts.tHash = DefaultTHash
|
|
|
|
}
|
|
|
|
ts.h = ts.tHash.Hash()
|
|
|
|
ts.h.Reset()
|
|
|
|
ts.first = true
|
|
|
|
ts.sums = FileInfoSums{}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2014-09-11 01:56:20 +00:00
|
|
|
func (ts *tarSum) Read(buf []byte) (int, error) {
|
2013-07-17 19:13:22 +00:00
|
|
|
if ts.finished {
|
2014-05-15 20:50:58 +00:00
|
|
|
return ts.bufWriter.Read(buf)
|
2013-07-17 19:13:22 +00:00
|
|
|
}
|
2014-09-25 22:58:35 +00:00
|
|
|
if len(ts.bufData) < len(buf) {
|
2014-09-06 12:29:19 +00:00
|
|
|
switch {
|
|
|
|
case len(buf) <= buf8K:
|
|
|
|
ts.bufData = make([]byte, buf8K)
|
|
|
|
case len(buf) <= buf16K:
|
|
|
|
ts.bufData = make([]byte, buf16K)
|
|
|
|
case len(buf) <= buf32K:
|
|
|
|
ts.bufData = make([]byte, buf32K)
|
|
|
|
default:
|
|
|
|
ts.bufData = make([]byte, len(buf))
|
|
|
|
}
|
2014-08-17 11:29:46 +00:00
|
|
|
}
|
2014-09-25 22:58:35 +00:00
|
|
|
buf2 := ts.bufData[:len(buf)]
|
2013-07-17 19:13:22 +00:00
|
|
|
|
|
|
|
n, err := ts.tarR.Read(buf2)
|
|
|
|
if err != nil {
|
|
|
|
if err == io.EOF {
|
|
|
|
if _, err := ts.h.Write(buf2[:n]); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
if !ts.first {
|
2014-09-04 20:13:50 +00:00
|
|
|
ts.sums = append(ts.sums, fileInfoSum{name: ts.currentFile, sum: hex.EncodeToString(ts.h.Sum(nil)), pos: ts.fileCounter})
|
|
|
|
ts.fileCounter++
|
2013-07-17 19:13:22 +00:00
|
|
|
ts.h.Reset()
|
|
|
|
} else {
|
|
|
|
ts.first = false
|
|
|
|
}
|
|
|
|
|
|
|
|
currentHeader, err := ts.tarR.Next()
|
|
|
|
if err != nil {
|
|
|
|
if err == io.EOF {
|
2014-09-16 06:52:06 +00:00
|
|
|
if err := ts.tarW.Close(); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
2014-05-15 20:50:58 +00:00
|
|
|
if _, err := io.Copy(ts.writer, ts.bufTar); err != nil {
|
2014-09-16 06:52:06 +00:00
|
|
|
return 0, err
|
|
|
|
}
|
2014-05-15 20:50:58 +00:00
|
|
|
if err := ts.writer.Close(); err != nil {
|
2013-07-17 19:13:22 +00:00
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
ts.finished = true
|
|
|
|
return n, nil
|
|
|
|
}
|
|
|
|
return n, err
|
|
|
|
}
|
2013-12-27 00:02:15 +00:00
|
|
|
ts.currentFile = strings.TrimSuffix(strings.TrimPrefix(currentHeader.Name, "./"), "/")
|
2013-07-17 19:13:22 +00:00
|
|
|
if err := ts.encodeHeader(currentHeader); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
if err := ts.tarW.WriteHeader(currentHeader); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
if _, err := ts.tarW.Write(buf2[:n]); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
ts.tarW.Flush()
|
2014-05-15 20:50:58 +00:00
|
|
|
if _, err := io.Copy(ts.writer, ts.bufTar); err != nil {
|
2013-07-17 19:13:22 +00:00
|
|
|
return 0, err
|
|
|
|
}
|
2014-05-15 20:50:58 +00:00
|
|
|
ts.writer.Flush()
|
2013-07-17 19:13:22 +00:00
|
|
|
|
2014-05-15 20:50:58 +00:00
|
|
|
return ts.bufWriter.Read(buf)
|
2013-07-17 19:13:22 +00:00
|
|
|
}
|
|
|
|
return n, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Filling the hash buffer
|
|
|
|
if _, err = ts.h.Write(buf2[:n]); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Filling the tar writter
|
|
|
|
if _, err = ts.tarW.Write(buf2[:n]); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
ts.tarW.Flush()
|
|
|
|
|
2014-05-15 20:50:58 +00:00
|
|
|
// Filling the output writer
|
|
|
|
if _, err = io.Copy(ts.writer, ts.bufTar); err != nil {
|
2013-07-17 19:13:22 +00:00
|
|
|
return 0, err
|
|
|
|
}
|
2014-05-15 20:50:58 +00:00
|
|
|
ts.writer.Flush()
|
2013-07-17 19:13:22 +00:00
|
|
|
|
2014-05-15 20:50:58 +00:00
|
|
|
return ts.bufWriter.Read(buf)
|
2013-07-17 19:13:22 +00:00
|
|
|
}
|
|
|
|
|
2014-09-11 01:56:20 +00:00
|
|
|
func (ts *tarSum) Sum(extra []byte) string {
|
2014-09-04 20:13:50 +00:00
|
|
|
ts.sums.SortBySums()
|
2014-05-15 20:50:58 +00:00
|
|
|
h := ts.tHash.Hash()
|
2013-07-22 23:16:31 +00:00
|
|
|
if extra != nil {
|
|
|
|
h.Write(extra)
|
|
|
|
}
|
2014-09-04 20:13:50 +00:00
|
|
|
for _, fis := range ts.sums {
|
|
|
|
log.Debugf("-->%s<--", fis.Sum())
|
|
|
|
h.Write([]byte(fis.Sum()))
|
2013-07-17 19:13:22 +00:00
|
|
|
}
|
2014-05-15 20:50:58 +00:00
|
|
|
checksum := ts.Version().String() + "+" + ts.tHash.Name() + ":" + hex.EncodeToString(h.Sum(nil))
|
2014-09-09 17:31:11 +00:00
|
|
|
log.Debugf("checksum processed: %s", checksum)
|
2013-07-17 19:13:22 +00:00
|
|
|
return checksum
|
|
|
|
}
|
2013-12-27 00:01:36 +00:00
|
|
|
|
2014-09-04 20:13:50 +00:00
|
|
|
func (ts *tarSum) GetSums() FileInfoSums {
|
2013-12-27 00:01:36 +00:00
|
|
|
return ts.sums
|
|
|
|
}
|