From 448c8ecb0fb453bb29fcada5e7486355ccfadbe9 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Wed, 21 May 2014 17:13:53 -0400 Subject: [PATCH 1/2] tarsum: include xattr headers in to the checksum this is to enhance the tarsum algorithm, but _MUST_ be done in lock step with the same for docker-registry. (PR will be cited) Docker-DCO-1.1-Signed-off-by: Vincent Batts (github: vbatts) --- pkg/tarsum/tarsum.go | 12 ++++++++++++ pkg/tarsum/tarsum_test.go | 5 +++++ pkg/tarsum/testdata/xattr/json | 1 + pkg/tarsum/testdata/xattr/layer.tar | Bin 0 -> 2560 bytes 4 files changed, 18 insertions(+) create mode 100644 pkg/tarsum/testdata/xattr/json create mode 100644 pkg/tarsum/testdata/xattr/layer.tar diff --git a/pkg/tarsum/tarsum.go b/pkg/tarsum/tarsum.go index 1399dd5d1f..4852e88371 100644 --- a/pkg/tarsum/tarsum.go +++ b/pkg/tarsum/tarsum.go @@ -95,6 +95,18 @@ func (ts *tarSum) encodeHeader(h *tar.Header) error { return err } } + + // include the additional pax headers, from an ordered list + var keys []string + for k := range h.Xattrs { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if _, err := ts.h.Write([]byte(k + h.Xattrs[k])); err != nil { + return err + } + } return nil } diff --git a/pkg/tarsum/tarsum_test.go b/pkg/tarsum/tarsum_test.go index b921ad284d..de9341173c 100644 --- a/pkg/tarsum/tarsum_test.go +++ b/pkg/tarsum/tarsum_test.go @@ -37,6 +37,11 @@ var testLayers = []testLayer{ jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json", gzip: true, tarsum: "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b"}, + { + filename: "testdata/xattr/layer.tar", + jsonfile: "testdata/xattr/json", + // without xattr support, this layer produces "tarsum+sha256:e86f81a4d552f13039b1396ed03ca968ea9717581f9577ef1876ea6ff9b38c98" + tarsum: "tarsum+sha256:eadcd9c55164e2a942ea67f781abbf645ee55d36e173c34de8a36d4d69d3021c"}, { filename: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar", jsonfile: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json", diff --git a/pkg/tarsum/testdata/xattr/json b/pkg/tarsum/testdata/xattr/json new file mode 100644 index 0000000000..328ea31fd9 --- /dev/null +++ b/pkg/tarsum/testdata/xattr/json @@ -0,0 +1 @@ +{"id":"4439c3c7f847954100b42b267e7e5529cac1d6934db082f65795c5ca2e594d93","parent":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","created":"2014-05-16T17:19:44.091534414Z","container":"5f92fb06cc58f357f0cde41394e2bbbb664e663974b2ac1693ab07b7a306749b","container_config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","setcap 'cap_setgid,cap_setuid+ep' ./file \u0026\u0026 getcap ./file"],"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.11.1-dev","config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":0} \ No newline at end of file diff --git a/pkg/tarsum/testdata/xattr/layer.tar b/pkg/tarsum/testdata/xattr/layer.tar new file mode 100644 index 0000000000000000000000000000000000000000..819351d42f41430bf402b3c672b571e3eaf8b095 GIT binary patch literal 2560 zcmWGYtnf%pOi3*&)-yCRu(Z@q%gjk-pe`_g00tbifq}UpgQ1bJv8joPiJ_SpgMp!u zxuGe8LIri5f!(yy;*!K7pwqC+5-*DGIpTE=s7h0FgVE5xzPBd+@To)G|2840byWhU|?oqf;;~Mb02E{2kHRk de~R-YhD)#rjPU%AB}7JrMnhmU1V%^*0091(G-Ch& literal 0 HcmV?d00001 From 0ecafb8a6909f5d342246aefb9ac97350cef6558 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 15 Sep 2014 16:08:05 -0400 Subject: [PATCH 2/2] tarsum: version the addition of xattrs Now that TarSum can be versioned, move the addition of xattr headers in to the next version of TarSum Signed-off-by: Vincent Batts --- pkg/tarsum/tarsum.go | 20 +++++++++++--------- pkg/tarsum/tarsum_test.go | 11 +++++++++-- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/pkg/tarsum/tarsum.go b/pkg/tarsum/tarsum.go index 4852e88371..bee77a832c 100644 --- a/pkg/tarsum/tarsum.go +++ b/pkg/tarsum/tarsum.go @@ -81,7 +81,7 @@ func (ts tarSum) selectHeaders(h *tar.Header, v Version) (set [][2]string) { {"devmajor", strconv.Itoa(int(h.Devmajor))}, {"devminor", strconv.Itoa(int(h.Devminor))}, } { - if v == VersionDev && elem[0] == "mtime" { + if v >= VersionDev && elem[0] == "mtime" { continue } set = append(set, elem) @@ -97,14 +97,16 @@ func (ts *tarSum) encodeHeader(h *tar.Header) error { } // include the additional pax headers, from an ordered list - var keys []string - for k := range h.Xattrs { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - if _, err := ts.h.Write([]byte(k + h.Xattrs[k])); err != nil { - return err + if ts.Version() >= VersionDev { + var keys []string + for k := range h.Xattrs { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if _, err := ts.h.Write([]byte(k + h.Xattrs[k])); err != nil { + return err + } } } return nil diff --git a/pkg/tarsum/tarsum_test.go b/pkg/tarsum/tarsum_test.go index de9341173c..7686dd80c9 100644 --- a/pkg/tarsum/tarsum_test.go +++ b/pkg/tarsum/tarsum_test.go @@ -38,10 +38,17 @@ var testLayers = []testLayer{ gzip: true, tarsum: "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b"}, { + // Tests existing version of TarSum when xattrs are present filename: "testdata/xattr/layer.tar", jsonfile: "testdata/xattr/json", - // without xattr support, this layer produces "tarsum+sha256:e86f81a4d552f13039b1396ed03ca968ea9717581f9577ef1876ea6ff9b38c98" - tarsum: "tarsum+sha256:eadcd9c55164e2a942ea67f781abbf645ee55d36e173c34de8a36d4d69d3021c"}, + version: Version0, + tarsum: "tarsum+sha256:e86f81a4d552f13039b1396ed03ca968ea9717581f9577ef1876ea6ff9b38c98"}, + { + // Tests next version of TarSum when xattrs are present + filename: "testdata/xattr/layer.tar", + jsonfile: "testdata/xattr/json", + version: VersionDev, + tarsum: "tarsum.dev+sha256:6235cd3a2afb7501bac541772a3d61a3634e95bc90bb39a4676e2cb98d08390d"}, { filename: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar", jsonfile: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json",