mirror of
https://github.com/moby/moby.git
synced 2022-11-09 12:21:53 -05:00
Merge pull request #5822 from creack/update_vendors
Update archive/tar vendored
This commit is contained in:
commit
eb61a1f73a
10 changed files with 811 additions and 25 deletions
|
@ -137,3 +137,32 @@ func TestTypeXGlobalHeaderDoesNotFail(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Some tar have both GNU specific (huge uid) and Ustar specific (long name) things.
|
||||
// Not supposed to happen (should use PAX instead of Ustar for long name) but it does and it should still work.
|
||||
func TestUntarUstarGnuConflict(t *testing.T) {
|
||||
f, err := os.Open("testdata/broken.tar")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
tr := tar.NewReader(f)
|
||||
// Iterate through the files in the archive.
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
// end of tar archive
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hdr.Name == "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("%s not found in the archive", "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm")
|
||||
}
|
||||
}
|
||||
|
|
BIN
archive/testdata/broken.tar
vendored
Normal file
BIN
archive/testdata/broken.tar
vendored
Normal file
Binary file not shown.
|
@ -53,7 +53,7 @@ clone hg code.google.com/p/gosqlite 74691fb6f837
|
|||
|
||||
# get Go tip's archive/tar, for xattr support
|
||||
# TODO after Go 1.3 drops, bump our minimum supported version and drop this vendored dep
|
||||
clone hg code.google.com/p/go a15f344a9efa
|
||||
clone hg code.google.com/p/go 3458ba248590
|
||||
mv src/code.google.com/p/go/src/pkg/archive/tar tmp-tar
|
||||
rm -rf src/code.google.com/p/go
|
||||
mkdir -p src/code.google.com/p/go/src/pkg/archive
|
||||
|
|
|
@ -38,6 +38,7 @@ const (
|
|||
TypeXGlobalHeader = 'g' // global extended header
|
||||
TypeGNULongName = 'L' // Next file has a long name
|
||||
TypeGNULongLink = 'K' // Next file symlinks to a file w/ a long name
|
||||
TypeGNUSparse = 'S' // sparse file
|
||||
)
|
||||
|
||||
// A Header represents a single header in a tar archive.
|
||||
|
|
|
@ -29,12 +29,57 @@ const maxNanoSecondIntSize = 9
|
|||
// The Next method advances to the next file in the archive (including the first),
|
||||
// and then it can be treated as an io.Reader to access the file's data.
|
||||
type Reader struct {
|
||||
r io.Reader
|
||||
err error
|
||||
nb int64 // number of unread bytes for current file entry
|
||||
pad int64 // amount of padding (ignored) after current file entry
|
||||
r io.Reader
|
||||
err error
|
||||
pad int64 // amount of padding (ignored) after current file entry
|
||||
curr numBytesReader // reader for current file entry
|
||||
}
|
||||
|
||||
// A numBytesReader is an io.Reader with a numBytes method, returning the number
|
||||
// of bytes remaining in the underlying encoded data.
|
||||
type numBytesReader interface {
|
||||
io.Reader
|
||||
numBytes() int64
|
||||
}
|
||||
|
||||
// A regFileReader is a numBytesReader for reading file data from a tar archive.
|
||||
type regFileReader struct {
|
||||
r io.Reader // underlying reader
|
||||
nb int64 // number of unread bytes for current file entry
|
||||
}
|
||||
|
||||
// A sparseFileReader is a numBytesReader for reading sparse file data from a tar archive.
|
||||
type sparseFileReader struct {
|
||||
rfr *regFileReader // reads the sparse-encoded file data
|
||||
sp []sparseEntry // the sparse map for the file
|
||||
pos int64 // keeps track of file position
|
||||
tot int64 // total size of the file
|
||||
}
|
||||
|
||||
// Keywords for GNU sparse files in a PAX extended header
|
||||
const (
|
||||
paxGNUSparseNumBlocks = "GNU.sparse.numblocks"
|
||||
paxGNUSparseOffset = "GNU.sparse.offset"
|
||||
paxGNUSparseNumBytes = "GNU.sparse.numbytes"
|
||||
paxGNUSparseMap = "GNU.sparse.map"
|
||||
paxGNUSparseName = "GNU.sparse.name"
|
||||
paxGNUSparseMajor = "GNU.sparse.major"
|
||||
paxGNUSparseMinor = "GNU.sparse.minor"
|
||||
paxGNUSparseSize = "GNU.sparse.size"
|
||||
paxGNUSparseRealSize = "GNU.sparse.realsize"
|
||||
)
|
||||
|
||||
// Keywords for old GNU sparse headers
|
||||
const (
|
||||
oldGNUSparseMainHeaderOffset = 386
|
||||
oldGNUSparseMainHeaderIsExtendedOffset = 482
|
||||
oldGNUSparseMainHeaderNumEntries = 4
|
||||
oldGNUSparseExtendedHeaderIsExtendedOffset = 504
|
||||
oldGNUSparseExtendedHeaderNumEntries = 21
|
||||
oldGNUSparseOffsetSize = 12
|
||||
oldGNUSparseNumBytesSize = 12
|
||||
)
|
||||
|
||||
// NewReader creates a new Reader reading from r.
|
||||
func NewReader(r io.Reader) *Reader { return &Reader{r: r} }
|
||||
|
||||
|
@ -64,6 +109,18 @@ func (tr *Reader) Next() (*Header, error) {
|
|||
tr.skipUnread()
|
||||
hdr = tr.readHeader()
|
||||
mergePAX(hdr, headers)
|
||||
|
||||
// Check for a PAX format sparse file
|
||||
sp, err := tr.checkForGNUSparsePAXHeaders(hdr, headers)
|
||||
if err != nil {
|
||||
tr.err = err
|
||||
return nil, err
|
||||
}
|
||||
if sp != nil {
|
||||
// Current file is a PAX format GNU sparse file.
|
||||
// Set the current file reader to a sparse file reader.
|
||||
tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size}
|
||||
}
|
||||
return hdr, nil
|
||||
case TypeGNULongName:
|
||||
// We have a GNU long name header. Its contents are the real file name.
|
||||
|
@ -87,6 +144,67 @@ func (tr *Reader) Next() (*Header, error) {
|
|||
return hdr, tr.err
|
||||
}
|
||||
|
||||
// checkForGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. If they are found, then
|
||||
// this function reads the sparse map and returns it. Unknown sparse formats are ignored, causing the file to
|
||||
// be treated as a regular file.
|
||||
func (tr *Reader) checkForGNUSparsePAXHeaders(hdr *Header, headers map[string]string) ([]sparseEntry, error) {
|
||||
var sparseFormat string
|
||||
|
||||
// Check for sparse format indicators
|
||||
major, majorOk := headers[paxGNUSparseMajor]
|
||||
minor, minorOk := headers[paxGNUSparseMinor]
|
||||
sparseName, sparseNameOk := headers[paxGNUSparseName]
|
||||
_, sparseMapOk := headers[paxGNUSparseMap]
|
||||
sparseSize, sparseSizeOk := headers[paxGNUSparseSize]
|
||||
sparseRealSize, sparseRealSizeOk := headers[paxGNUSparseRealSize]
|
||||
|
||||
// Identify which, if any, sparse format applies from which PAX headers are set
|
||||
if majorOk && minorOk {
|
||||
sparseFormat = major + "." + minor
|
||||
} else if sparseNameOk && sparseMapOk {
|
||||
sparseFormat = "0.1"
|
||||
} else if sparseSizeOk {
|
||||
sparseFormat = "0.0"
|
||||
} else {
|
||||
// Not a PAX format GNU sparse file.
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Check for unknown sparse format
|
||||
if sparseFormat != "0.0" && sparseFormat != "0.1" && sparseFormat != "1.0" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Update hdr from GNU sparse PAX headers
|
||||
if sparseNameOk {
|
||||
hdr.Name = sparseName
|
||||
}
|
||||
if sparseSizeOk {
|
||||
realSize, err := strconv.ParseInt(sparseSize, 10, 0)
|
||||
if err != nil {
|
||||
return nil, ErrHeader
|
||||
}
|
||||
hdr.Size = realSize
|
||||
} else if sparseRealSizeOk {
|
||||
realSize, err := strconv.ParseInt(sparseRealSize, 10, 0)
|
||||
if err != nil {
|
||||
return nil, ErrHeader
|
||||
}
|
||||
hdr.Size = realSize
|
||||
}
|
||||
|
||||
// Set up the sparse map, according to the particular sparse format in use
|
||||
var sp []sparseEntry
|
||||
var err error
|
||||
switch sparseFormat {
|
||||
case "0.0", "0.1":
|
||||
sp, err = readGNUSparseMap0x1(headers)
|
||||
case "1.0":
|
||||
sp, err = readGNUSparseMap1x0(tr.curr)
|
||||
}
|
||||
return sp, err
|
||||
}
|
||||
|
||||
// mergePAX merges well known headers according to PAX standard.
|
||||
// In general headers with the same name as those found
|
||||
// in the header struct overwrite those found in the header
|
||||
|
@ -194,6 +312,11 @@ func parsePAX(r io.Reader) (map[string]string, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// For GNU PAX sparse format 0.0 support.
|
||||
// This function transforms the sparse format 0.0 headers into sparse format 0.1 headers.
|
||||
var sparseMap bytes.Buffer
|
||||
|
||||
headers := make(map[string]string)
|
||||
// Each record is constructed as
|
||||
// "%d %s=%s\n", length, keyword, value
|
||||
|
@ -211,7 +334,7 @@ func parsePAX(r io.Reader) (map[string]string, error) {
|
|||
return nil, ErrHeader
|
||||
}
|
||||
// Extract everything between the decimal and the n -1 on the
|
||||
// beginning to to eat the ' ', -1 on the end to skip the newline.
|
||||
// beginning to eat the ' ', -1 on the end to skip the newline.
|
||||
var record []byte
|
||||
record, buf = buf[sp+1:n-1], buf[n:]
|
||||
// The first equals is guaranteed to mark the end of the key.
|
||||
|
@ -221,7 +344,21 @@ func parsePAX(r io.Reader) (map[string]string, error) {
|
|||
return nil, ErrHeader
|
||||
}
|
||||
key, value := record[:eq], record[eq+1:]
|
||||
headers[string(key)] = string(value)
|
||||
|
||||
keyStr := string(key)
|
||||
if keyStr == paxGNUSparseOffset || keyStr == paxGNUSparseNumBytes {
|
||||
// GNU sparse format 0.0 special key. Write to sparseMap instead of using the headers map.
|
||||
sparseMap.Write(value)
|
||||
sparseMap.Write([]byte{','})
|
||||
} else {
|
||||
// Normal key. Set the value in the headers map.
|
||||
headers[keyStr] = string(value)
|
||||
}
|
||||
}
|
||||
if sparseMap.Len() != 0 {
|
||||
// Add sparse info to headers, chopping off the extra comma
|
||||
sparseMap.Truncate(sparseMap.Len() - 1)
|
||||
headers[paxGNUSparseMap] = sparseMap.String()
|
||||
}
|
||||
return headers, nil
|
||||
}
|
||||
|
@ -268,8 +405,8 @@ func (tr *Reader) octal(b []byte) int64 {
|
|||
|
||||
// skipUnread skips any unread bytes in the existing file entry, as well as any alignment padding.
|
||||
func (tr *Reader) skipUnread() {
|
||||
nr := tr.nb + tr.pad // number of bytes to skip
|
||||
tr.nb, tr.pad = 0, 0
|
||||
nr := tr.numBytes() + tr.pad // number of bytes to skip
|
||||
tr.curr, tr.pad = nil, 0
|
||||
if sr, ok := tr.r.(io.Seeker); ok {
|
||||
if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil {
|
||||
return
|
||||
|
@ -331,14 +468,14 @@ func (tr *Reader) readHeader() *Header {
|
|||
// so its magic bytes, like the rest of the block, are NULs.
|
||||
magic := string(s.next(8)) // contains version field as well.
|
||||
var format string
|
||||
switch magic {
|
||||
case "ustar\x0000": // POSIX tar (1003.1-1988)
|
||||
switch {
|
||||
case magic[:6] == "ustar\x00": // POSIX tar (1003.1-1988)
|
||||
if string(header[508:512]) == "tar\x00" {
|
||||
format = "star"
|
||||
} else {
|
||||
format = "posix"
|
||||
}
|
||||
case "ustar \x00": // old GNU tar
|
||||
case magic == "ustar \x00": // old GNU tar
|
||||
format = "gnu"
|
||||
}
|
||||
|
||||
|
@ -373,30 +510,308 @@ func (tr *Reader) readHeader() *Header {
|
|||
|
||||
// Maximum value of hdr.Size is 64 GB (12 octal digits),
|
||||
// so there's no risk of int64 overflowing.
|
||||
tr.nb = int64(hdr.Size)
|
||||
tr.pad = -tr.nb & (blockSize - 1) // blockSize is a power of two
|
||||
nb := int64(hdr.Size)
|
||||
tr.pad = -nb & (blockSize - 1) // blockSize is a power of two
|
||||
|
||||
// Set the current file reader.
|
||||
tr.curr = ®FileReader{r: tr.r, nb: nb}
|
||||
|
||||
// Check for old GNU sparse format entry.
|
||||
if hdr.Typeflag == TypeGNUSparse {
|
||||
// Get the real size of the file.
|
||||
hdr.Size = tr.octal(header[483:495])
|
||||
|
||||
// Read the sparse map.
|
||||
sp := tr.readOldGNUSparseMap(header)
|
||||
if tr.err != nil {
|
||||
return nil
|
||||
}
|
||||
// Current file is a GNU sparse file. Update the current file reader.
|
||||
tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size}
|
||||
}
|
||||
|
||||
return hdr
|
||||
}
|
||||
|
||||
// A sparseEntry holds a single entry in a sparse file's sparse map.
|
||||
// A sparse entry indicates the offset and size in a sparse file of a
|
||||
// block of data.
|
||||
type sparseEntry struct {
|
||||
offset int64
|
||||
numBytes int64
|
||||
}
|
||||
|
||||
// readOldGNUSparseMap reads the sparse map as stored in the old GNU sparse format.
|
||||
// The sparse map is stored in the tar header if it's small enough. If it's larger than four entries,
|
||||
// then one or more extension headers are used to store the rest of the sparse map.
|
||||
func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry {
|
||||
isExtended := header[oldGNUSparseMainHeaderIsExtendedOffset] != 0
|
||||
spCap := oldGNUSparseMainHeaderNumEntries
|
||||
if isExtended {
|
||||
spCap += oldGNUSparseExtendedHeaderNumEntries
|
||||
}
|
||||
sp := make([]sparseEntry, 0, spCap)
|
||||
s := slicer(header[oldGNUSparseMainHeaderOffset:])
|
||||
|
||||
// Read the four entries from the main tar header
|
||||
for i := 0; i < oldGNUSparseMainHeaderNumEntries; i++ {
|
||||
offset := tr.octal(s.next(oldGNUSparseOffsetSize))
|
||||
numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize))
|
||||
if tr.err != nil {
|
||||
tr.err = ErrHeader
|
||||
return nil
|
||||
}
|
||||
if offset == 0 && numBytes == 0 {
|
||||
break
|
||||
}
|
||||
sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
|
||||
}
|
||||
|
||||
for isExtended {
|
||||
// There are more entries. Read an extension header and parse its entries.
|
||||
sparseHeader := make([]byte, blockSize)
|
||||
if _, tr.err = io.ReadFull(tr.r, sparseHeader); tr.err != nil {
|
||||
return nil
|
||||
}
|
||||
isExtended = sparseHeader[oldGNUSparseExtendedHeaderIsExtendedOffset] != 0
|
||||
s = slicer(sparseHeader)
|
||||
for i := 0; i < oldGNUSparseExtendedHeaderNumEntries; i++ {
|
||||
offset := tr.octal(s.next(oldGNUSparseOffsetSize))
|
||||
numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize))
|
||||
if tr.err != nil {
|
||||
tr.err = ErrHeader
|
||||
return nil
|
||||
}
|
||||
if offset == 0 && numBytes == 0 {
|
||||
break
|
||||
}
|
||||
sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
|
||||
}
|
||||
}
|
||||
return sp
|
||||
}
|
||||
|
||||
// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format version 1.0.
|
||||
// The sparse map is stored just before the file data and padded out to the nearest block boundary.
|
||||
func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) {
|
||||
buf := make([]byte, 2*blockSize)
|
||||
sparseHeader := buf[:blockSize]
|
||||
|
||||
// readDecimal is a helper function to read a decimal integer from the sparse map
|
||||
// while making sure to read from the file in blocks of size blockSize
|
||||
readDecimal := func() (int64, error) {
|
||||
// Look for newline
|
||||
nl := bytes.IndexByte(sparseHeader, '\n')
|
||||
if nl == -1 {
|
||||
if len(sparseHeader) >= blockSize {
|
||||
// This is an error
|
||||
return 0, ErrHeader
|
||||
}
|
||||
oldLen := len(sparseHeader)
|
||||
newLen := oldLen + blockSize
|
||||
if cap(sparseHeader) < newLen {
|
||||
// There's more header, but we need to make room for the next block
|
||||
copy(buf, sparseHeader)
|
||||
sparseHeader = buf[:newLen]
|
||||
} else {
|
||||
// There's more header, and we can just reslice
|
||||
sparseHeader = sparseHeader[:newLen]
|
||||
}
|
||||
|
||||
// Now that sparseHeader is large enough, read next block
|
||||
if _, err := io.ReadFull(r, sparseHeader[oldLen:newLen]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Look for a newline in the new data
|
||||
nl = bytes.IndexByte(sparseHeader[oldLen:newLen], '\n')
|
||||
if nl == -1 {
|
||||
// This is an error
|
||||
return 0, ErrHeader
|
||||
}
|
||||
nl += oldLen // We want the position from the beginning
|
||||
}
|
||||
// Now that we've found a newline, read a number
|
||||
n, err := strconv.ParseInt(string(sparseHeader[:nl]), 10, 0)
|
||||
if err != nil {
|
||||
return 0, ErrHeader
|
||||
}
|
||||
|
||||
// Update sparseHeader to consume this number
|
||||
sparseHeader = sparseHeader[nl+1:]
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Read the first block
|
||||
if _, err := io.ReadFull(r, sparseHeader); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The first line contains the number of entries
|
||||
numEntries, err := readDecimal()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Read all the entries
|
||||
sp := make([]sparseEntry, 0, numEntries)
|
||||
for i := int64(0); i < numEntries; i++ {
|
||||
// Read the offset
|
||||
offset, err := readDecimal()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Read numBytes
|
||||
numBytes, err := readDecimal()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
|
||||
}
|
||||
|
||||
return sp, nil
|
||||
}
|
||||
|
||||
// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format version 0.1.
|
||||
// The sparse map is stored in the PAX headers.
|
||||
func readGNUSparseMap0x1(headers map[string]string) ([]sparseEntry, error) {
|
||||
// Get number of entries
|
||||
numEntriesStr, ok := headers[paxGNUSparseNumBlocks]
|
||||
if !ok {
|
||||
return nil, ErrHeader
|
||||
}
|
||||
numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0)
|
||||
if err != nil {
|
||||
return nil, ErrHeader
|
||||
}
|
||||
|
||||
sparseMap := strings.Split(headers[paxGNUSparseMap], ",")
|
||||
|
||||
// There should be two numbers in sparseMap for each entry
|
||||
if int64(len(sparseMap)) != 2*numEntries {
|
||||
return nil, ErrHeader
|
||||
}
|
||||
|
||||
// Loop through the entries in the sparse map
|
||||
sp := make([]sparseEntry, 0, numEntries)
|
||||
for i := int64(0); i < numEntries; i++ {
|
||||
offset, err := strconv.ParseInt(sparseMap[2*i], 10, 0)
|
||||
if err != nil {
|
||||
return nil, ErrHeader
|
||||
}
|
||||
numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 0)
|
||||
if err != nil {
|
||||
return nil, ErrHeader
|
||||
}
|
||||
sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
|
||||
}
|
||||
|
||||
return sp, nil
|
||||
}
|
||||
|
||||
// numBytes returns the number of bytes left to read in the current file's entry
|
||||
// in the tar archive, or 0 if there is no current file.
|
||||
func (tr *Reader) numBytes() int64 {
|
||||
if tr.curr == nil {
|
||||
// No current file, so no bytes
|
||||
return 0
|
||||
}
|
||||
return tr.curr.numBytes()
|
||||
}
|
||||
|
||||
// Read reads from the current entry in the tar archive.
|
||||
// It returns 0, io.EOF when it reaches the end of that entry,
|
||||
// until Next is called to advance to the next entry.
|
||||
func (tr *Reader) Read(b []byte) (n int, err error) {
|
||||
if tr.nb == 0 {
|
||||
if tr.curr == nil {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n, err = tr.curr.Read(b)
|
||||
if err != nil && err != io.EOF {
|
||||
tr.err = err
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (rfr *regFileReader) Read(b []byte) (n int, err error) {
|
||||
if rfr.nb == 0 {
|
||||
// file consumed
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
if int64(len(b)) > tr.nb {
|
||||
b = b[0:tr.nb]
|
||||
if int64(len(b)) > rfr.nb {
|
||||
b = b[0:rfr.nb]
|
||||
}
|
||||
n, err = tr.r.Read(b)
|
||||
tr.nb -= int64(n)
|
||||
n, err = rfr.r.Read(b)
|
||||
rfr.nb -= int64(n)
|
||||
|
||||
if err == io.EOF && tr.nb > 0 {
|
||||
if err == io.EOF && rfr.nb > 0 {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
tr.err = err
|
||||
return
|
||||
}
|
||||
|
||||
// numBytes returns the number of bytes left to read in the file's data in the tar archive.
|
||||
func (rfr *regFileReader) numBytes() int64 {
|
||||
return rfr.nb
|
||||
}
|
||||
|
||||
// readHole reads a sparse file hole ending at offset toOffset
|
||||
func (sfr *sparseFileReader) readHole(b []byte, toOffset int64) int {
|
||||
n64 := toOffset - sfr.pos
|
||||
if n64 > int64(len(b)) {
|
||||
n64 = int64(len(b))
|
||||
}
|
||||
n := int(n64)
|
||||
for i := 0; i < n; i++ {
|
||||
b[i] = 0
|
||||
}
|
||||
sfr.pos += n64
|
||||
return n
|
||||
}
|
||||
|
||||
// Read reads the sparse file data in expanded form.
|
||||
func (sfr *sparseFileReader) Read(b []byte) (n int, err error) {
|
||||
if len(sfr.sp) == 0 {
|
||||
// No more data fragments to read from.
|
||||
if sfr.pos < sfr.tot {
|
||||
// We're in the last hole
|
||||
n = sfr.readHole(b, sfr.tot)
|
||||
return
|
||||
}
|
||||
// Otherwise, we're at the end of the file
|
||||
return 0, io.EOF
|
||||
}
|
||||
if sfr.pos < sfr.sp[0].offset {
|
||||
// We're in a hole
|
||||
n = sfr.readHole(b, sfr.sp[0].offset)
|
||||
return
|
||||
}
|
||||
|
||||
// We're not in a hole, so we'll read from the next data fragment
|
||||
posInFragment := sfr.pos - sfr.sp[0].offset
|
||||
bytesLeft := sfr.sp[0].numBytes - posInFragment
|
||||
if int64(len(b)) > bytesLeft {
|
||||
b = b[0:bytesLeft]
|
||||
}
|
||||
|
||||
n, err = sfr.rfr.Read(b)
|
||||
sfr.pos += int64(n)
|
||||
|
||||
if int64(n) == bytesLeft {
|
||||
// We're done with this fragment
|
||||
sfr.sp = sfr.sp[1:]
|
||||
}
|
||||
|
||||
if err == io.EOF && sfr.pos < sfr.tot {
|
||||
// We reached the end of the last fragment's data, but there's a final hole
|
||||
err = nil
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// numBytes returns the number of bytes left to read in the sparse file's
|
||||
// sparse-encoded data in the tar archive.
|
||||
func (sfr *sparseFileReader) numBytes() int64 {
|
||||
return sfr.rfr.nb
|
||||
}
|
||||
|
|
|
@ -9,6 +9,7 @@ import (
|
|||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
@ -54,8 +55,92 @@ var gnuTarTest = &untarTest{
|
|||
},
|
||||
}
|
||||
|
||||
var sparseTarTest = &untarTest{
|
||||
file: "testdata/sparse-formats.tar",
|
||||
headers: []*Header{
|
||||
{
|
||||
Name: "sparse-gnu",
|
||||
Mode: 420,
|
||||
Uid: 1000,
|
||||
Gid: 1000,
|
||||
Size: 200,
|
||||
ModTime: time.Unix(1392395740, 0),
|
||||
Typeflag: 0x53,
|
||||
Linkname: "",
|
||||
Uname: "david",
|
||||
Gname: "david",
|
||||
Devmajor: 0,
|
||||
Devminor: 0,
|
||||
},
|
||||
{
|
||||
Name: "sparse-posix-0.0",
|
||||
Mode: 420,
|
||||
Uid: 1000,
|
||||
Gid: 1000,
|
||||
Size: 200,
|
||||
ModTime: time.Unix(1392342187, 0),
|
||||
Typeflag: 0x30,
|
||||
Linkname: "",
|
||||
Uname: "david",
|
||||
Gname: "david",
|
||||
Devmajor: 0,
|
||||
Devminor: 0,
|
||||
},
|
||||
{
|
||||
Name: "sparse-posix-0.1",
|
||||
Mode: 420,
|
||||
Uid: 1000,
|
||||
Gid: 1000,
|
||||
Size: 200,
|
||||
ModTime: time.Unix(1392340456, 0),
|
||||
Typeflag: 0x30,
|
||||
Linkname: "",
|
||||
Uname: "david",
|
||||
Gname: "david",
|
||||
Devmajor: 0,
|
||||
Devminor: 0,
|
||||
},
|
||||
{
|
||||
Name: "sparse-posix-1.0",
|
||||
Mode: 420,
|
||||
Uid: 1000,
|
||||
Gid: 1000,
|
||||
Size: 200,
|
||||
ModTime: time.Unix(1392337404, 0),
|
||||
Typeflag: 0x30,
|
||||
Linkname: "",
|
||||
Uname: "david",
|
||||
Gname: "david",
|
||||
Devmajor: 0,
|
||||
Devminor: 0,
|
||||
},
|
||||
{
|
||||
Name: "end",
|
||||
Mode: 420,
|
||||
Uid: 1000,
|
||||
Gid: 1000,
|
||||
Size: 4,
|
||||
ModTime: time.Unix(1392398319, 0),
|
||||
Typeflag: 0x30,
|
||||
Linkname: "",
|
||||
Uname: "david",
|
||||
Gname: "david",
|
||||
Devmajor: 0,
|
||||
Devminor: 0,
|
||||
},
|
||||
},
|
||||
cksums: []string{
|
||||
"6f53234398c2449fe67c1812d993012f",
|
||||
"6f53234398c2449fe67c1812d993012f",
|
||||
"6f53234398c2449fe67c1812d993012f",
|
||||
"6f53234398c2449fe67c1812d993012f",
|
||||
"b0061974914468de549a2af8ced10316",
|
||||
},
|
||||
}
|
||||
|
||||
var untarTests = []*untarTest{
|
||||
gnuTarTest,
|
||||
sparseTarTest,
|
||||
{
|
||||
file: "testdata/star.tar",
|
||||
headers: []*Header{
|
||||
|
@ -386,7 +471,7 @@ func TestParsePAXHeader(t *testing.T) {
|
|||
func TestParsePAXTime(t *testing.T) {
|
||||
// Some valid PAX time values
|
||||
timestamps := map[string]time.Time{
|
||||
"1350244992.023960108": time.Unix(1350244992, 23960108), // The commoon case
|
||||
"1350244992.023960108": time.Unix(1350244992, 23960108), // The common case
|
||||
"1350244992.02396010": time.Unix(1350244992, 23960100), // Lower precision value
|
||||
"1350244992.0239601089": time.Unix(1350244992, 23960108), // Higher precision value
|
||||
"1350244992": time.Unix(1350244992, 0), // Low precision value
|
||||
|
@ -423,3 +508,236 @@ func TestMergePAX(t *testing.T) {
|
|||
t.Errorf("incorrect merge: got %+v, want %+v", hdr, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSparseEndToEnd(t *testing.T) {
|
||||
test := sparseTarTest
|
||||
f, err := os.Open(test.file)
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
tr := NewReader(f)
|
||||
|
||||
headers := test.headers
|
||||
cksums := test.cksums
|
||||
nread := 0
|
||||
|
||||
// loop over all files
|
||||
for ; ; nread++ {
|
||||
hdr, err := tr.Next()
|
||||
if hdr == nil || err == io.EOF {
|
||||
break
|
||||
}
|
||||
|
||||
// check the header
|
||||
if !reflect.DeepEqual(*hdr, *headers[nread]) {
|
||||
t.Errorf("Incorrect header:\nhave %+v\nwant %+v",
|
||||
*hdr, headers[nread])
|
||||
}
|
||||
|
||||
// read and checksum the file data
|
||||
h := md5.New()
|
||||
_, err = io.Copy(h, tr)
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error: %v", err)
|
||||
}
|
||||
|
||||
// verify checksum
|
||||
have := fmt.Sprintf("%x", h.Sum(nil))
|
||||
want := cksums[nread]
|
||||
if want != have {
|
||||
t.Errorf("Bad checksum on file %s:\nhave %+v\nwant %+v", hdr.Name, have, want)
|
||||
}
|
||||
}
|
||||
if nread != len(headers) {
|
||||
t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(headers), nread)
|
||||
}
|
||||
}
|
||||
|
||||
type sparseFileReadTest struct {
|
||||
sparseData []byte
|
||||
sparseMap []sparseEntry
|
||||
realSize int64
|
||||
expected []byte
|
||||
}
|
||||
|
||||
var sparseFileReadTests = []sparseFileReadTest{
|
||||
{
|
||||
sparseData: []byte("abcde"),
|
||||
sparseMap: []sparseEntry{
|
||||
{offset: 0, numBytes: 2},
|
||||
{offset: 5, numBytes: 3},
|
||||
},
|
||||
realSize: 8,
|
||||
expected: []byte("ab\x00\x00\x00cde"),
|
||||
},
|
||||
{
|
||||
sparseData: []byte("abcde"),
|
||||
sparseMap: []sparseEntry{
|
||||
{offset: 0, numBytes: 2},
|
||||
{offset: 5, numBytes: 3},
|
||||
},
|
||||
realSize: 10,
|
||||
expected: []byte("ab\x00\x00\x00cde\x00\x00"),
|
||||
},
|
||||
{
|
||||
sparseData: []byte("abcde"),
|
||||
sparseMap: []sparseEntry{
|
||||
{offset: 1, numBytes: 3},
|
||||
{offset: 6, numBytes: 2},
|
||||
},
|
||||
realSize: 8,
|
||||
expected: []byte("\x00abc\x00\x00de"),
|
||||
},
|
||||
{
|
||||
sparseData: []byte("abcde"),
|
||||
sparseMap: []sparseEntry{
|
||||
{offset: 1, numBytes: 3},
|
||||
{offset: 6, numBytes: 2},
|
||||
},
|
||||
realSize: 10,
|
||||
expected: []byte("\x00abc\x00\x00de\x00\x00"),
|
||||
},
|
||||
{
|
||||
sparseData: []byte(""),
|
||||
sparseMap: nil,
|
||||
realSize: 2,
|
||||
expected: []byte("\x00\x00"),
|
||||
},
|
||||
}
|
||||
|
||||
func TestSparseFileReader(t *testing.T) {
|
||||
for i, test := range sparseFileReadTests {
|
||||
r := bytes.NewReader(test.sparseData)
|
||||
nb := int64(r.Len())
|
||||
sfr := &sparseFileReader{
|
||||
rfr: ®FileReader{r: r, nb: nb},
|
||||
sp: test.sparseMap,
|
||||
pos: 0,
|
||||
tot: test.realSize,
|
||||
}
|
||||
if sfr.numBytes() != nb {
|
||||
t.Errorf("test %d: Before reading, sfr.numBytes() = %d, want %d", i, sfr.numBytes(), nb)
|
||||
}
|
||||
buf, err := ioutil.ReadAll(sfr)
|
||||
if err != nil {
|
||||
t.Errorf("test %d: Unexpected error: %v", i, err)
|
||||
}
|
||||
if e := test.expected; !bytes.Equal(buf, e) {
|
||||
t.Errorf("test %d: Contents = %v, want %v", i, buf, e)
|
||||
}
|
||||
if sfr.numBytes() != 0 {
|
||||
t.Errorf("test %d: After draining the reader, numBytes() was nonzero", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSparseIncrementalRead(t *testing.T) {
|
||||
sparseMap := []sparseEntry{{10, 2}}
|
||||
sparseData := []byte("Go")
|
||||
expected := "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Go\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
|
||||
r := bytes.NewReader(sparseData)
|
||||
nb := int64(r.Len())
|
||||
sfr := &sparseFileReader{
|
||||
rfr: ®FileReader{r: r, nb: nb},
|
||||
sp: sparseMap,
|
||||
pos: 0,
|
||||
tot: int64(len(expected)),
|
||||
}
|
||||
|
||||
// We'll read the data 6 bytes at a time, with a hole of size 10 at
|
||||
// the beginning and one of size 8 at the end.
|
||||
var outputBuf bytes.Buffer
|
||||
buf := make([]byte, 6)
|
||||
for {
|
||||
n, err := sfr.Read(buf)
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("Read: unexpected error %v\n", err)
|
||||
}
|
||||
if n > 0 {
|
||||
_, err := outputBuf.Write(buf[:n])
|
||||
if err != nil {
|
||||
t.Errorf("Write: unexpected error %v\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
got := outputBuf.String()
|
||||
if got != expected {
|
||||
t.Errorf("Contents = %v, want %v", got, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadGNUSparseMap0x1(t *testing.T) {
|
||||
headers := map[string]string{
|
||||
paxGNUSparseNumBlocks: "4",
|
||||
paxGNUSparseMap: "0,5,10,5,20,5,30,5",
|
||||
}
|
||||
expected := []sparseEntry{
|
||||
{offset: 0, numBytes: 5},
|
||||
{offset: 10, numBytes: 5},
|
||||
{offset: 20, numBytes: 5},
|
||||
{offset: 30, numBytes: 5},
|
||||
}
|
||||
|
||||
sp, err := readGNUSparseMap0x1(headers)
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(sp, expected) {
|
||||
t.Errorf("Incorrect sparse map: got %v, wanted %v", sp, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadGNUSparseMap1x0(t *testing.T) {
|
||||
// This test uses lots of holes so the sparse header takes up more than two blocks
|
||||
numEntries := 100
|
||||
expected := make([]sparseEntry, 0, numEntries)
|
||||
sparseMap := new(bytes.Buffer)
|
||||
|
||||
fmt.Fprintf(sparseMap, "%d\n", numEntries)
|
||||
for i := 0; i < numEntries; i++ {
|
||||
offset := int64(2048 * i)
|
||||
numBytes := int64(1024)
|
||||
expected = append(expected, sparseEntry{offset: offset, numBytes: numBytes})
|
||||
fmt.Fprintf(sparseMap, "%d\n%d\n", offset, numBytes)
|
||||
}
|
||||
|
||||
// Make the header the smallest multiple of blockSize that fits the sparseMap
|
||||
headerBlocks := (sparseMap.Len() + blockSize - 1) / blockSize
|
||||
bufLen := blockSize * headerBlocks
|
||||
buf := make([]byte, bufLen)
|
||||
copy(buf, sparseMap.Bytes())
|
||||
|
||||
// Get an reader to read the sparse map
|
||||
r := bytes.NewReader(buf)
|
||||
|
||||
// Read the sparse map
|
||||
sp, err := readGNUSparseMap1x0(r)
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(sp, expected) {
|
||||
t.Errorf("Incorrect sparse map: got %v, wanted %v", sp, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUninitializedRead(t *testing.T) {
|
||||
test := gnuTarTest
|
||||
f, err := os.Open(test.file)
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
tr := NewReader(f)
|
||||
_, err = tr.Read([]byte{})
|
||||
if err == nil || err != io.EOF {
|
||||
t.Errorf("Unexpected error: %v, wanted %v", err, io.EOF)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
BIN
vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/sparse-formats.tar
vendored
Normal file
BIN
vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/sparse-formats.tar
vendored
Normal file
Binary file not shown.
BIN
vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/writer-big-long.tar
vendored
Normal file
BIN
vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/writer-big-long.tar
vendored
Normal file
Binary file not shown.
|
@ -218,8 +218,8 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
|
|||
tw.cString(prefixHeaderBytes, prefix, false, paxNone, nil)
|
||||
|
||||
// Use the ustar magic if we used ustar long names.
|
||||
if len(prefix) > 0 {
|
||||
copy(header[257:265], []byte("ustar\000"))
|
||||
if len(prefix) > 0 && !tw.usedBinary {
|
||||
copy(header[257:265], []byte("ustar\x00"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -103,6 +103,29 @@ var writerTests = []*writerTest{
|
|||
},
|
||||
},
|
||||
},
|
||||
// The truncated test file was produced using these commands:
|
||||
// dd if=/dev/zero bs=1048576 count=16384 > (longname/)*15 /16gig.txt
|
||||
// tar -b 1 -c -f- (longname/)*15 /16gig.txt | dd bs=512 count=8 > writer-big-long.tar
|
||||
{
|
||||
file: "testdata/writer-big-long.tar",
|
||||
entries: []*writerTestEntry{
|
||||
{
|
||||
header: &Header{
|
||||
Name: strings.Repeat("longname/", 15) + "16gig.txt",
|
||||
Mode: 0644,
|
||||
Uid: 1000,
|
||||
Gid: 1000,
|
||||
Size: 16 << 30,
|
||||
ModTime: time.Unix(1399583047, 0),
|
||||
Typeflag: '0',
|
||||
Uname: "guillaume",
|
||||
Gname: "guillaume",
|
||||
},
|
||||
// fake contents
|
||||
contents: strings.Repeat("\x00", 4<<10),
|
||||
},
|
||||
},
|
||||
},
|
||||
// This file was produced using gnu tar 1.17
|
||||
// gnutar -b 4 --format=ustar (longname/)*15 + file.txt
|
||||
{
|
||||
|
|
Loading…
Add table
Reference in a new issue