From 1f61084d83aea37b212468aaa975020094b7f7c9 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Thu, 23 Jul 2015 14:19:58 -0700 Subject: [PATCH] Fix uses of "int" where "int64" should be used instead Some structures use int for sizes and UNIX timestamps. On some platforms, int is 32 bits, so this can lead to the year 2038 issues and overflows when dealing with large containers or layers. Consistently use int64 to store sizes and UNIX timestamps in api/types/types.go. Update related to code accordingly (i.e. strconv.FormatInt instead of strconv.Itoa). Use int64 in progressreader package to avoid integer overflow when dealing with large quantities. Update related code accordingly. Signed-off-by: Aaron Lehmann --- api/client/build.go | 2 +- api/client/ps/custom_test.go | 2 +- api/types/types.go | 12 ++++++------ builder/internals.go | 2 +- builder/job.go | 4 ++-- builder/job_test.go | 8 ++++---- daemon/list.go | 6 +++--- graph/graph.go | 6 +++--- graph/graph_unix.go | 2 +- graph/graph_windows.go | 2 +- graph/import.go | 2 +- graph/list.go | 12 ++++++------ graph/pull_v1.go | 4 ++-- graph/pull_v2.go | 4 ++-- graph/push_v1.go | 2 +- graph/push_v2.go | 2 +- pkg/jsonmessage/jsonmessage.go | 6 +++--- pkg/progressreader/progressreader.go | 12 ++++++------ pkg/progressreader/progressreader_test.go | 4 ++-- pkg/tarsum/versioning.go | 10 +++++----- registry/registry_test.go | 2 +- registry/session.go | 6 +++--- 22 files changed, 56 insertions(+), 56 deletions(-) diff --git a/api/client/build.go b/api/client/build.go index bd4f9cff5c..6dc59aab2e 100644 --- a/api/client/build.go +++ b/api/client/build.go @@ -462,7 +462,7 @@ func getContextFromURL(out io.Writer, remoteURL, dockerfileName string) (absCont In: response.Body, Out: out, Formatter: streamformatter.NewStreamFormatter(), - Size: int(response.ContentLength), + Size: response.ContentLength, NewLines: true, ID: "", Action: fmt.Sprintf("Downloading build context from remote url: %s", remoteURL), diff --git a/api/client/ps/custom_test.go b/api/client/ps/custom_test.go index e657537536..8676518e57 100644 --- a/api/client/ps/custom_test.go +++ b/api/client/ps/custom_test.go @@ -27,7 +27,7 @@ func TestContainerContextID(t *testing.T) { {types.Container{Image: "ubuntu"}, true, "ubuntu", imageHeader, ctx.Image}, {types.Container{Image: ""}, true, "", imageHeader, ctx.Image}, {types.Container{Command: "sh -c 'ls -la'"}, true, `"sh -c 'ls -la'"`, commandHeader, ctx.Command}, - {types.Container{Created: int(unix)}, true, time.Unix(unix, 0).String(), createdAtHeader, ctx.CreatedAt}, + {types.Container{Created: unix}, true, time.Unix(unix, 0).String(), createdAtHeader, ctx.CreatedAt}, {types.Container{Ports: []types.Port{{PrivatePort: 8080, PublicPort: 8080, Type: "tcp"}}}, true, "8080/tcp", portsHeader, ctx.Ports}, {types.Container{Status: "RUNNING"}, true, "RUNNING", statusHeader, ctx.Status}, {types.Container{SizeRw: 10}, true, "10 B", sizeHeader, ctx.Size}, diff --git a/api/types/types.go b/api/types/types.go index 844ddd0f24..a4e0ccf874 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -70,9 +70,9 @@ type Image struct { ParentId string RepoTags []string RepoDigests []string - Created int - Size int - VirtualSize int + Created int64 + Size int64 + VirtualSize int64 Labels map[string]string } @@ -112,10 +112,10 @@ type Container struct { Names []string Image string Command string - Created int + Created int64 Ports []Port - SizeRw int `json:",omitempty"` - SizeRootFs int `json:",omitempty"` + SizeRw int64 `json:",omitempty"` + SizeRootFs int64 `json:",omitempty"` Labels map[string]string Status string HostConfig struct { diff --git a/builder/internals.go b/builder/internals.go index b23b49e8f0..6fac572939 100644 --- a/builder/internals.go +++ b/builder/internals.go @@ -330,7 +330,7 @@ func calcCopyInfo(b *builder, cmdName string, cInfos *[]*copyInfo, origPath stri In: resp.Body, Out: b.OutOld, Formatter: b.StreamFormatter, - Size: int(resp.ContentLength), + Size: resp.ContentLength, NewLines: true, ID: "", Action: "Downloading", diff --git a/builder/job.go b/builder/job.go index d3b51aaecc..5e3a3c8bdf 100644 --- a/builder/job.go +++ b/builder/job.go @@ -139,7 +139,7 @@ func Build(d *daemon.Daemon, buildConfig *Config) error { } defer f.Body.Close() ct := f.Header.Get("Content-Type") - clen := int(f.ContentLength) + clen := f.ContentLength contentType, bodyReader, err := inspectResponse(ct, f.Body, clen) defer bodyReader.Close() @@ -316,7 +316,7 @@ func Commit(name string, d *daemon.Daemon, c *CommitConfig) (string, error) { // - an io.Reader for the response body // - an error value which will be non-nil either when something goes wrong while // reading bytes from r or when the detected content-type is not acceptable. -func inspectResponse(ct string, r io.ReadCloser, clen int) (string, io.ReadCloser, error) { +func inspectResponse(ct string, r io.ReadCloser, clen int64) (string, io.ReadCloser, error) { plen := clen if plen <= 0 || plen > maxPreambleLength { plen = maxPreambleLength diff --git a/builder/job_test.go b/builder/job_test.go index 9ce74b2251..2f432d70fd 100644 --- a/builder/job_test.go +++ b/builder/job_test.go @@ -31,7 +31,7 @@ func TestInspectEmptyResponse(t *testing.T) { func TestInspectResponseBinary(t *testing.T) { ct := "application/octet-stream" br := ioutil.NopCloser(bytes.NewReader(binaryContext)) - contentType, bReader, err := inspectResponse(ct, br, len(binaryContext)) + contentType, bReader, err := inspectResponse(ct, br, int64(len(binaryContext))) if err != nil { t.Fatal(err) } @@ -56,7 +56,7 @@ func TestResponseUnsupportedContentType(t *testing.T) { content := []byte(textPlainDockerfile) ct := "application/json" br := ioutil.NopCloser(bytes.NewReader(content)) - contentType, bReader, err := inspectResponse(ct, br, len(textPlainDockerfile)) + contentType, bReader, err := inspectResponse(ct, br, int64(len(textPlainDockerfile))) if err == nil { t.Fatal("Should have returned an error on content-type 'application/json'") @@ -77,7 +77,7 @@ func TestInspectResponseTextSimple(t *testing.T) { content := []byte(textPlainDockerfile) ct := "text/plain" br := ioutil.NopCloser(bytes.NewReader(content)) - contentType, bReader, err := inspectResponse(ct, br, len(content)) + contentType, bReader, err := inspectResponse(ct, br, int64(len(content))) if err != nil { t.Fatal(err) } @@ -96,7 +96,7 @@ func TestInspectResponseTextSimple(t *testing.T) { func TestInspectResponseEmptyContentType(t *testing.T) { content := []byte(textPlainDockerfile) br := ioutil.NopCloser(bytes.NewReader(content)) - contentType, bodyReader, err := inspectResponse("", br, len(content)) + contentType, bodyReader, err := inspectResponse("", br, int64(len(content))) if err != nil { t.Fatal(err) } diff --git a/daemon/list.go b/daemon/list.go index 14e333a7c5..1cf7da4c9f 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -152,7 +152,7 @@ func (daemon *Daemon) Containers(config *ContainersConfig) ([]*types.Container, } else { newC.Command = fmt.Sprintf("%s", container.Path) } - newC.Created = int(container.Created.Unix()) + newC.Created = container.Created.Unix() newC.Status = container.State.String() newC.HostConfig.NetworkMode = string(container.hostConfig.NetworkMode) @@ -185,8 +185,8 @@ func (daemon *Daemon) Containers(config *ContainersConfig) ([]*types.Container, if config.Size { sizeRw, sizeRootFs := container.GetSize() - newC.SizeRw = int(sizeRw) - newC.SizeRootFs = int(sizeRootFs) + newC.SizeRw = sizeRw + newC.SizeRootFs = sizeRootFs } newC.Labels = container.Config.Labels containers = append(containers, newC) diff --git a/graph/graph.go b/graph/graph.go index daf088dae0..0594959223 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -190,7 +190,7 @@ func (graph *Graph) Get(name string) (*image.Image, error) { } img.Size = size - if err := graph.saveSize(graph.imageRoot(id), int(img.Size)); err != nil { + if err := graph.saveSize(graph.imageRoot(id), img.Size); err != nil { return nil, err } } @@ -490,8 +490,8 @@ func (graph *Graph) loadImage(id string) (*image.Image, error) { } // saveSize stores the `size` in the provided graph `img` directory `root`. -func (graph *Graph) saveSize(root string, size int) error { - if err := ioutil.WriteFile(filepath.Join(root, layersizeFileName), []byte(strconv.Itoa(size)), 0600); err != nil { +func (graph *Graph) saveSize(root string, size int64) error { + if err := ioutil.WriteFile(filepath.Join(root, layersizeFileName), []byte(strconv.FormatInt(size, 10)), 0600); err != nil { return fmt.Errorf("Error storing image size in %s/%s: %s", root, layersizeFileName, err) } return nil diff --git a/graph/graph_unix.go b/graph/graph_unix.go index a07807d221..f03dfd1cfa 100644 --- a/graph/graph_unix.go +++ b/graph/graph_unix.go @@ -95,7 +95,7 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader } } - if err := graph.saveSize(root, int(img.Size)); err != nil { + if err := graph.saveSize(root, img.Size); err != nil { return err } diff --git a/graph/graph_windows.go b/graph/graph_windows.go index a19b356f73..df41f50a92 100644 --- a/graph/graph_windows.go +++ b/graph/graph_windows.go @@ -95,7 +95,7 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader } } - if err := graph.saveSize(root, int(img.Size)); err != nil { + if err := graph.saveSize(root, img.Size); err != nil { return err } diff --git a/graph/import.go b/graph/import.go index 4e09cf0296..5c11a76caf 100644 --- a/graph/import.go +++ b/graph/import.go @@ -45,7 +45,7 @@ func (s *TagStore) Import(src string, repo string, tag string, inConfig io.ReadC In: resp.Body, Out: outStream, Formatter: sf, - Size: int(resp.ContentLength), + Size: resp.ContentLength, NewLines: true, ID: "", Action: "Importing", diff --git a/graph/list.go b/graph/list.go index 72eae63427..13ed73873b 100644 --- a/graph/list.go +++ b/graph/list.go @@ -99,9 +99,9 @@ func (s *TagStore) Images(filterArgs, filter string, all bool) ([]*types.Image, newImage := new(types.Image) newImage.ParentId = image.Parent newImage.ID = image.ID - newImage.Created = int(image.Created.Unix()) - newImage.Size = int(image.Size) - newImage.VirtualSize = int(s.graph.GetParentsSize(image) + image.Size) + newImage.Created = image.Created.Unix() + newImage.Size = image.Size + newImage.VirtualSize = s.graph.GetParentsSize(image) + image.Size newImage.Labels = image.ContainerConfig.Labels if utils.DigestReference(ref) { @@ -136,9 +136,9 @@ func (s *TagStore) Images(filterArgs, filter string, all bool) ([]*types.Image, newImage.RepoTags = []string{":"} newImage.RepoDigests = []string{"@"} newImage.ID = image.ID - newImage.Created = int(image.Created.Unix()) - newImage.Size = int(image.Size) - newImage.VirtualSize = int(s.graph.GetParentsSize(image) + image.Size) + newImage.Created = image.Created.Unix() + newImage.Size = image.Size + newImage.VirtualSize = s.graph.GetParentsSize(image) + image.Size newImage.Labels = image.ContainerConfig.Labels images = append(images, newImage) diff --git a/graph/pull_v1.go b/graph/pull_v1.go index 5eb75372ff..0280f1f950 100644 --- a/graph/pull_v1.go +++ b/graph/pull_v1.go @@ -256,7 +256,7 @@ func (p *v1Puller) pullImage(imgID, endpoint string, token []string) (bool, erro out.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Pulling metadata", nil)) var ( imgJSON []byte - imgSize int + imgSize int64 err error img *image.Image ) @@ -290,7 +290,7 @@ func (p *v1Puller) pullImage(imgID, endpoint string, token []string) (bool, erro status = fmt.Sprintf("Pulling fs layer [retries: %d]", j) } out.Write(p.sf.FormatProgress(stringid.TruncateID(id), status, nil)) - layer, err := p.session.GetRemoteImageLayer(img.ID, endpoint, int64(imgSize)) + layer, err := p.session.GetRemoteImageLayer(img.ID, endpoint, imgSize) if uerr, ok := err.(*url.Error); ok { err = uerr.Err } diff --git a/graph/pull_v2.go b/graph/pull_v2.go index 6c2fa20653..d98614109d 100644 --- a/graph/pull_v2.go +++ b/graph/pull_v2.go @@ -167,7 +167,7 @@ func (p *v2Puller) download(di *downloadInfo) { In: ioutil.NopCloser(io.TeeReader(layerDownload, verifier)), Out: out, Formatter: p.sf, - Size: int(di.size), + Size: di.size, NewLines: false, ID: stringid.TruncateID(di.img.ID), Action: "Downloading", @@ -264,7 +264,7 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (bool, error) { In: d.tmpFile, Out: out, Formatter: p.sf, - Size: int(d.size), + Size: d.size, NewLines: false, ID: stringid.TruncateID(d.img.ID), Action: "Extracting", diff --git a/graph/push_v1.go b/graph/push_v1.go index d473040f88..6de293403b 100644 --- a/graph/push_v1.go +++ b/graph/push_v1.go @@ -289,7 +289,7 @@ func (p *v1Pusher) pushImage(imgID, ep string, token []string) (checksum string, In: layerData, Out: p.out, Formatter: p.sf, - Size: int(layerData.Size), + Size: layerData.Size, NewLines: false, ID: stringid.TruncateID(imgData.ID), Action: "Pushing", diff --git a/graph/push_v2.go b/graph/push_v2.go index 869347df6e..3d74ad46f8 100644 --- a/graph/push_v2.go +++ b/graph/push_v2.go @@ -239,7 +239,7 @@ func (p *v2Pusher) pushV2Image(bs distribution.BlobService, img *image.Image) (d In: ioutil.NopCloser(tf), Out: out, Formatter: p.sf, - Size: int(size), + Size: size, NewLines: false, ID: stringid.TruncateID(img.ID), Action: "Pushing", diff --git a/pkg/jsonmessage/jsonmessage.go b/pkg/jsonmessage/jsonmessage.go index 7db1626e48..0c93ee6704 100644 --- a/pkg/jsonmessage/jsonmessage.go +++ b/pkg/jsonmessage/jsonmessage.go @@ -23,8 +23,8 @@ func (e *JSONError) Error() string { type JSONProgress struct { terminalFd uintptr - Current int `json:"current,omitempty"` - Total int `json:"total,omitempty"` + Current int64 `json:"current,omitempty"` + Total int64 `json:"total,omitempty"` Start int64 `json:"start,omitempty"` } @@ -64,7 +64,7 @@ func (p *JSONProgress) String() string { numbersBox = fmt.Sprintf("%8v/%v", current, total) if p.Current > 0 && p.Start > 0 && percentage < 50 { - fromStart := time.Now().UTC().Sub(time.Unix(int64(p.Start), 0)) + fromStart := time.Now().UTC().Sub(time.Unix(p.Start, 0)) perEntry := fromStart / time.Duration(p.Current) left := time.Duration(p.Total-p.Current) * perEntry left = (left / time.Second) * time.Second diff --git a/pkg/progressreader/progressreader.go b/pkg/progressreader/progressreader.go index d55fadd718..f48442b591 100644 --- a/pkg/progressreader/progressreader.go +++ b/pkg/progressreader/progressreader.go @@ -14,9 +14,9 @@ type Config struct { In io.ReadCloser // Stream to read from Out io.Writer // Where to send progress bar to Formatter *streamformatter.StreamFormatter - Size int - Current int - LastUpdate int + Size int64 + Current int64 + LastUpdate int64 NewLines bool ID string Action string @@ -29,11 +29,11 @@ func New(newReader Config) *Config { func (config *Config) Read(p []byte) (n int, err error) { read, err := config.In.Read(p) - config.Current += read - updateEvery := 1024 * 512 //512kB + config.Current += int64(read) + updateEvery := int64(1024 * 512) //512kB if config.Size > 0 { // Update progress for every 1% read if 1% < 512kB - if increment := int(0.01 * float64(config.Size)); increment < updateEvery { + if increment := int64(0.01 * float64(config.Size)); increment < updateEvery { updateEvery = increment } } diff --git a/pkg/progressreader/progressreader_test.go b/pkg/progressreader/progressreader_test.go index fdf40cbb35..21d9b0f057 100644 --- a/pkg/progressreader/progressreader_test.go +++ b/pkg/progressreader/progressreader_test.go @@ -20,7 +20,7 @@ func TestOutputOnPrematureClose(t *testing.T) { In: reader, Out: writer, Formatter: streamformatter.NewStreamFormatter(), - Size: len(content), + Size: int64(len(content)), NewLines: true, ID: "Test", Action: "Read", @@ -60,7 +60,7 @@ func TestCompleteSilently(t *testing.T) { In: reader, Out: writer, Formatter: streamformatter.NewStreamFormatter(), - Size: len(content), + Size: int64(len(content)), NewLines: true, ID: "Test", Action: "Read", diff --git a/pkg/tarsum/versioning.go b/pkg/tarsum/versioning.go index 8988b9f587..2882286854 100644 --- a/pkg/tarsum/versioning.go +++ b/pkg/tarsum/versioning.go @@ -95,17 +95,17 @@ func (f tarHeaderSelectFunc) selectHeaders(h *tar.Header) (orderedHeaders [][2]s func v0TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) { return [][2]string{ {"name", h.Name}, - {"mode", strconv.Itoa(int(h.Mode))}, + {"mode", strconv.FormatInt(h.Mode, 10)}, {"uid", strconv.Itoa(h.Uid)}, {"gid", strconv.Itoa(h.Gid)}, - {"size", strconv.Itoa(int(h.Size))}, - {"mtime", strconv.Itoa(int(h.ModTime.UTC().Unix()))}, + {"size", strconv.FormatInt(h.Size, 10)}, + {"mtime", strconv.FormatInt(h.ModTime.UTC().Unix(), 10)}, {"typeflag", string([]byte{h.Typeflag})}, {"linkname", h.Linkname}, {"uname", h.Uname}, {"gname", h.Gname}, - {"devmajor", strconv.Itoa(int(h.Devmajor))}, - {"devminor", strconv.Itoa(int(h.Devminor))}, + {"devmajor", strconv.FormatInt(h.Devmajor, 10)}, + {"devminor", strconv.FormatInt(h.Devminor, 10)}, } } diff --git a/registry/registry_test.go b/registry/registry_test.go index 88b08dffae..d9ac5c6f20 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -185,7 +185,7 @@ func TestGetRemoteImageJSON(t *testing.T) { if err != nil { t.Fatal(err) } - assertEqual(t, size, 154, "Expected size 154") + assertEqual(t, size, int64(154), "Expected size 154") if len(json) <= 0 { t.Fatal("Expected non-empty json") } diff --git a/registry/session.go b/registry/session.go index 9bec7c1b2a..a9c4daf3aa 100644 --- a/registry/session.go +++ b/registry/session.go @@ -240,7 +240,7 @@ func (r *Session) LookupRemoteImage(imgID, registry string) error { } // GetRemoteImageJSON retrieves an image's JSON metadata from the registry. -func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int, error) { +func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int64, error) { res, err := r.client.Get(registry + "images/" + imgID + "/json") if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -250,9 +250,9 @@ func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int, error return nil, -1, httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } // if the size header is not present, then set it to '-1' - imageSize := -1 + imageSize := int64(-1) if hdr := res.Header.Get("X-Docker-Size"); hdr != "" { - imageSize, err = strconv.Atoi(hdr) + imageSize, err = strconv.ParseInt(hdr, 10, 64) if err != nil { return nil, -1, err }