From c30a55f14dbbe3971ba0ac716ba69a60868f4490 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 29 Mar 2015 23:17:23 +0200 Subject: [PATCH] Refactor utils/utils, fixes #11923 Signed-off-by: Antonio Murdaca --- api/client/attach.go | 3 +- api/client/build.go | 2 +- api/client/client.go | 12 + api/client/exec.go | 5 +- api/client/history.go | 4 +- api/client/inspect.go | 5 +- api/client/ps.go | 4 +- api/client/run.go | 3 +- api/client/search.go | 4 +- api/client/start.go | 3 +- builder/internals.go | 4 +- builder/job.go | 4 +- daemon/attach.go | 46 +++- daemon/container.go | 6 +- daemon/daemon.go | 9 +- daemon/execdriver/lxc/driver.go | 4 +- daemon/execdriver/lxc/lxc_template.go | 4 +- daemon/execdriver/utils.go | 14 +- daemon/info.go | 3 +- daemon/utils_test.go | 3 +- docker/docker.go | 3 +- engine/env.go | 4 +- graph/graph.go | 3 +- graph/import.go | 3 +- graph/load.go | 3 +- image/image.go | 14 +- integration/graph_test.go | 3 +- integration/runtime_test.go | 3 +- integration/z_final_test.go | 5 +- pkg/fileutils/fileutils.go | 54 +++++ pkg/fileutils/fileutils_test.go | 81 +++++++ pkg/httputils/httputils.go | 26 ++ pkg/ioutils/readers.go | 10 + pkg/ioutils/writers.go | 21 ++ pkg/ioutils/writers_test.go | 41 ++++ pkg/requestdecorator/requestdecorator_test.go | 16 +- pkg/resolvconf/resolvconf.go | 4 +- pkg/stringutils/stringutils.go | 56 +++++ pkg/stringutils/stringutils_test.go | 29 +++ registry/config.go | 4 +- registry/session.go | 29 ++- registry/session_v2.go | 18 +- runconfig/hostconfig.go | 8 +- runconfig/parse.go | 7 +- utils/utils.go | 224 +----------------- utils/utils_test.go | 146 ++++-------- 46 files changed, 530 insertions(+), 427 deletions(-) create mode 100644 pkg/fileutils/fileutils_test.go create mode 100644 pkg/httputils/httputils.go create mode 100644 pkg/ioutils/writers_test.go diff --git a/api/client/attach.go b/api/client/attach.go index 48cb8b4478..77947a2941 100644 --- a/api/client/attach.go +++ b/api/client/attach.go @@ -9,7 +9,6 @@ import ( "github.com/docker/docker/engine" flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/signal" - "github.com/docker/docker/utils" ) // CmdAttach attaches to a running container. @@ -81,7 +80,7 @@ func (cli *DockerCli) CmdAttach(args ...string) error { return err } if status != 0 { - return &utils.StatusError{StatusCode: status} + return &StatusError{StatusCode: status} } return nil diff --git a/api/client/build.go b/api/client/build.go index f1bceb4a16..dc54c22ffa 100644 --- a/api/client/build.go +++ b/api/client/build.go @@ -302,7 +302,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if jerr.Code == 0 { jerr.Code = 1 } - return &utils.StatusError{Status: jerr.Message, StatusCode: jerr.Code} + return &StatusError{Status: jerr.Message, StatusCode: jerr.Code} } return err } diff --git a/api/client/client.go b/api/client/client.go index 4cfce5f684..c849fa40f6 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -3,3 +3,15 @@ // Run "docker help SUBCOMMAND" or "docker SUBCOMMAND --help" to see more information on any Docker subcommand, including the full list of options supported for the subcommand. // See https://docs.docker.com/installation/ for instructions on installing Docker. package client + +import "fmt" + +// An StatusError reports an unsuccessful exit by a command. +type StatusError struct { + Status string + StatusCode int +} + +func (e *StatusError) Error() string { + return fmt.Sprintf("Status: %s, Code: %d", e.Status, e.StatusCode) +} diff --git a/api/client/exec.go b/api/client/exec.go index 27e6878df4..25b7a85fd2 100644 --- a/api/client/exec.go +++ b/api/client/exec.go @@ -9,7 +9,6 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/pkg/promise" "github.com/docker/docker/runconfig" - "github.com/docker/docker/utils" ) // CmdExec runs a command in a running container. @@ -21,7 +20,7 @@ func (cli *DockerCli) CmdExec(args ...string) error { execConfig, err := runconfig.ParseExec(cmd, args) // just in case the ParseExec does not exit if execConfig.Container == "" || err != nil { - return &utils.StatusError{StatusCode: 1} + return &StatusError{StatusCode: 1} } stream, _, err := cli.call("POST", "/containers/"+execConfig.Container+"/exec", execConfig, nil) @@ -122,7 +121,7 @@ func (cli *DockerCli) CmdExec(args ...string) error { } if status != 0 { - return &utils.StatusError{StatusCode: status} + return &StatusError{StatusCode: status} } return nil diff --git a/api/client/history.go b/api/client/history.go index 844a6fb770..4ac46d92ce 100644 --- a/api/client/history.go +++ b/api/client/history.go @@ -9,8 +9,8 @@ import ( "github.com/docker/docker/api/types" flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/stringid" + "github.com/docker/docker/pkg/stringutils" "github.com/docker/docker/pkg/units" - "github.com/docker/docker/utils" ) // CmdHistory shows the history of an image. @@ -51,7 +51,7 @@ func (cli *DockerCli) CmdHistory(args ...string) error { if *noTrunc { fmt.Fprintf(w, "%s\t", entry.CreatedBy) } else { - fmt.Fprintf(w, "%s\t", utils.Trunc(entry.CreatedBy, 45)) + fmt.Fprintf(w, "%s\t", stringutils.Truncate(entry.CreatedBy, 45)) } fmt.Fprintf(w, "%s\t", units.HumanSize(float64(entry.Size))) fmt.Fprintf(w, "%s", entry.Comment) diff --git a/api/client/inspect.go b/api/client/inspect.go index 0f47480b14..8514b1ecbc 100644 --- a/api/client/inspect.go +++ b/api/client/inspect.go @@ -9,7 +9,6 @@ import ( "text/template" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/utils" ) // CmdInspect displays low-level information on one or more containers or images. @@ -27,7 +26,7 @@ func (cli *DockerCli) CmdInspect(args ...string) error { var err error if tmpl, err = template.New("").Funcs(funcMap).Parse(*tmplStr); err != nil { fmt.Fprintf(cli.err, "Template parsing error: %v\n", err) - return &utils.StatusError{StatusCode: 64, + return &StatusError{StatusCode: 64, Status: "Template parsing error: " + err.Error()} } } @@ -86,7 +85,7 @@ func (cli *DockerCli) CmdInspect(args ...string) error { } if status != 0 { - return &utils.StatusError{StatusCode: status} + return &StatusError{StatusCode: status} } return nil } diff --git a/api/client/ps.go b/api/client/ps.go index be20d7a6f6..44f5ff0d21 100644 --- a/api/client/ps.go +++ b/api/client/ps.go @@ -15,8 +15,8 @@ import ( flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/parsers/filters" "github.com/docker/docker/pkg/stringid" + "github.com/docker/docker/pkg/stringutils" "github.com/docker/docker/pkg/units" - "github.com/docker/docker/utils" ) // CmdPs outputs a list of Docker containers. @@ -135,7 +135,7 @@ func (cli *DockerCli) CmdPs(args ...string) error { ) if !*noTrunc { - command = utils.Trunc(command, 20) + command = stringutils.Truncate(command, 20) // only display the default name for the container with notrunc is passed for _, name := range names { diff --git a/api/client/run.go b/api/client/run.go index 474c88f981..b37b6bab29 100644 --- a/api/client/run.go +++ b/api/client/run.go @@ -12,7 +12,6 @@ import ( "github.com/docker/docker/pkg/resolvconf" "github.com/docker/docker/pkg/signal" "github.com/docker/docker/runconfig" - "github.com/docker/docker/utils" ) func (cid *cidFile) Close() error { @@ -242,7 +241,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { } } if status != 0 { - return &utils.StatusError{StatusCode: status} + return &StatusError{StatusCode: status} } return nil } diff --git a/api/client/search.go b/api/client/search.go index 8f4eb0b301..5e0a22f014 100644 --- a/api/client/search.go +++ b/api/client/search.go @@ -10,8 +10,8 @@ import ( flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/parsers" + "github.com/docker/docker/pkg/stringutils" "github.com/docker/docker/registry" - "github.com/docker/docker/utils" ) type ByStars []registry.SearchResult @@ -68,7 +68,7 @@ func (cli *DockerCli) CmdSearch(args ...string) error { desc := strings.Replace(res.Description, "\n", " ", -1) desc = strings.Replace(desc, "\r", " ", -1) if !*noTrunc && len(desc) > 45 { - desc = utils.Trunc(desc, 42) + "..." + desc = stringutils.Truncate(desc, 42) + "..." } fmt.Fprintf(w, "%s\t%s\t%d\t", res.Name, desc, res.StarCount) if res.IsOfficial { diff --git a/api/client/start.go b/api/client/start.go index 66aa5150db..a03b8c1d29 100644 --- a/api/client/start.go +++ b/api/client/start.go @@ -11,7 +11,6 @@ import ( flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/signal" - "github.com/docker/docker/utils" ) func (cli *DockerCli) forwardAllSignals(cid string) chan os.Signal { @@ -156,7 +155,7 @@ func (cli *DockerCli) CmdStart(args ...string) error { return err } if status != 0 { - return &utils.StatusError{StatusCode: status} + return &StatusError{StatusCode: status} } } return nil diff --git a/builder/internals.go b/builder/internals.go index e0c7987ca1..728ccde8ae 100644 --- a/builder/internals.go +++ b/builder/internals.go @@ -25,6 +25,7 @@ import ( imagepkg "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" + "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/parsers" @@ -35,7 +36,6 @@ import ( "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/runconfig" - "github.com/docker/docker/utils" ) func (b *Builder) readContext(context io.Reader) error { @@ -250,7 +250,7 @@ func calcCopyInfo(b *Builder, cmdName string, cInfos *[]*copyInfo, origPath stri *cInfos = append(*cInfos, &ci) // Initiate the download - resp, err := utils.Download(ci.origPath) + resp, err := httputils.Download(ci.origPath) if err != nil { return err } diff --git a/builder/job.go b/builder/job.go index 89ed52f873..b0ce8ddc09 100644 --- a/builder/job.go +++ b/builder/job.go @@ -16,12 +16,12 @@ import ( "github.com/docker/docker/engine" "github.com/docker/docker/graph" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" - "github.com/docker/docker/utils" ) // whitelist of commands allowed for a commit/import @@ -106,7 +106,7 @@ func (b *BuilderJob) CmdBuild(job *engine.Job) error { } context = c } else if urlutil.IsURL(remoteURL) { - f, err := utils.Download(remoteURL) + f, err := httputils.Download(remoteURL) if err != nil { return err } diff --git a/daemon/attach.go b/daemon/attach.go index f95de41d52..b2b8d09067 100644 --- a/daemon/attach.go +++ b/daemon/attach.go @@ -10,7 +10,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/jsonlog" "github.com/docker/docker/pkg/promise" - "github.com/docker/docker/utils" ) func (c *Container) AttachWithLogs(stdin io.ReadCloser, stdout, stderr io.Writer, logs, stream bool) error { @@ -131,7 +130,7 @@ func attach(streamConfig *StreamConfig, openStdin, stdinOnce, tty bool, stdin io var err error if tty { - _, err = utils.CopyEscapable(cStdin, stdin) + _, err = copyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) @@ -185,3 +184,46 @@ func attach(streamConfig *StreamConfig, openStdin, stdinOnce, tty bool, stdin io return nil }) } + +// Code c/c from io.Copy() modified to handle escape sequence +func copyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) { + buf := make([]byte, 32*1024) + for { + nr, er := src.Read(buf) + if nr > 0 { + // ---- Docker addition + // char 16 is C-p + if nr == 1 && buf[0] == 16 { + nr, er = src.Read(buf) + // char 17 is C-q + if nr == 1 && buf[0] == 17 { + if err := src.Close(); err != nil { + return 0, err + } + return 0, nil + } + } + // ---- End of docker + nw, ew := dst.Write(buf[0:nr]) + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er == io.EOF { + break + } + if er != nil { + err = er + break + } + } + return written, err +} diff --git a/daemon/container.go b/daemon/container.go index e831f07a52..99fe157ad8 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -1063,7 +1063,7 @@ func (container *Container) setupContainerDns() error { updatedResolvConf, modified := resolvconf.FilterResolvDns(latestResolvConf, container.daemon.config.Bridge.EnableIPv6) if modified { // changes have occurred during resolv.conf localhost cleanup: generate an updated hash - newHash, err := utils.HashData(bytes.NewReader(updatedResolvConf)) + newHash, err := ioutils.HashData(bytes.NewReader(updatedResolvConf)) if err != nil { return err } @@ -1118,7 +1118,7 @@ func (container *Container) setupContainerDns() error { } //get a sha256 hash of the resolv conf at this point so we can check //for changes when the host resolv.conf changes (e.g. network update) - resolvHash, err := utils.HashData(bytes.NewReader(resolvConf)) + resolvHash, err := ioutils.HashData(bytes.NewReader(resolvConf)) if err != nil { return err } @@ -1150,7 +1150,7 @@ func (container *Container) updateResolvConf(updatedResolvConf []byte, newResolv if err != nil { return err } - curHash, err := utils.HashData(bytes.NewReader(resolvBytes)) + curHash, err := ioutils.HashData(bytes.NewReader(resolvBytes)) if err != nil { return err } diff --git a/daemon/daemon.go b/daemon/daemon.go index 789cd2b32b..e2ca568052 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -32,6 +32,7 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/broadcastwriter" + "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/graphdb" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/namesgenerator" @@ -431,7 +432,7 @@ func (daemon *Daemon) setupResolvconfWatcher() error { updatedResolvConf, modified := resolvconf.FilterResolvDns(updatedResolvConf, daemon.config.Bridge.EnableIPv6) if modified { // changes have occurred during localhost cleanup: generate an updated hash - newHash, err := utils.HashData(bytes.NewReader(updatedResolvConf)) + newHash, err := ioutils.HashData(bytes.NewReader(updatedResolvConf)) if err != nil { logrus.Debugf("Error generating hash of new resolv.conf: %v", err) } else { @@ -830,7 +831,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine, registryService if err != nil { return nil, fmt.Errorf("Unable to get the TempDir under %s: %s", config.Root, err) } - realTmp, err := utils.ReadSymlinkedDirectory(tmp) + realTmp, err := fileutils.ReadSymlinkedDirectory(tmp) if err != nil { return nil, fmt.Errorf("Unable to get the full path to the TempDir (%s): %s", tmp, err) } @@ -841,7 +842,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine, registryService if _, err := os.Stat(config.Root); err != nil && os.IsNotExist(err) { realRoot = config.Root } else { - realRoot, err = utils.ReadSymlinkedDirectory(config.Root) + realRoot, err = fileutils.ReadSymlinkedDirectory(config.Root) if err != nil { return nil, fmt.Errorf("Unable to get the full path to root (%s): %s", config.Root, err) } @@ -959,7 +960,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine, registryService if err := os.Mkdir(path.Dir(localCopy), 0700); err != nil && !os.IsExist(err) { return nil, err } - if _, err := utils.CopyFile(sysInitPath, localCopy); err != nil { + if _, err := fileutils.CopyFile(sysInitPath, localCopy); err != nil { return nil, err } if err := os.Chmod(localCopy, 0700); err != nil { diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 97b34bb678..1637bc2c69 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -18,10 +18,10 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/execdriver" + "github.com/docker/docker/pkg/stringutils" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" "github.com/docker/docker/pkg/version" - "github.com/docker/docker/utils" "github.com/docker/libcontainer" "github.com/docker/libcontainer/cgroups" "github.com/docker/libcontainer/configs" @@ -187,7 +187,7 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba // without exec in go we have to do this horrible shell hack... shellString := "mount --make-rslave /; exec " + - utils.ShellQuoteArguments(params) + stringutils.ShellQuoteArguments(params) params = []string{ "unshare", "-m", "--", "/bin/sh", "-c", shellString, diff --git a/daemon/execdriver/lxc/lxc_template.go b/daemon/execdriver/lxc/lxc_template.go index 6d6decb79f..6c182ab394 100644 --- a/daemon/execdriver/lxc/lxc_template.go +++ b/daemon/execdriver/lxc/lxc_template.go @@ -9,7 +9,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/execdriver" nativeTemplate "github.com/docker/docker/daemon/execdriver/native/template" - "github.com/docker/docker/utils" + "github.com/docker/docker/pkg/stringutils" "github.com/docker/libcontainer/label" ) @@ -177,7 +177,7 @@ func keepCapabilities(adds []string, drops []string) ([]string, error) { } func dropList(drops []string) ([]string, error) { - if utils.StringsContainsNoCase(drops, "all") { + if stringutils.InSlice(drops, "all") { var newCaps []string for _, capName := range execdriver.GetAllCapabilities() { cap := execdriver.GetCapability(capName) diff --git a/daemon/execdriver/utils.go b/daemon/execdriver/utils.go index e1fc9b9014..407c4f4fa1 100644 --- a/daemon/execdriver/utils.go +++ b/daemon/execdriver/utils.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/docker/docker/utils" + "github.com/docker/docker/pkg/stringutils" "github.com/syndtr/gocapability/capability" ) @@ -89,17 +89,17 @@ func TweakCapabilities(basics, adds, drops []string) ([]string, error) { if strings.ToLower(cap) == "all" { continue } - if !utils.StringsContainsNoCase(allCaps, cap) { + if !stringutils.InSlice(allCaps, cap) { return nil, fmt.Errorf("Unknown capability drop: %q", cap) } } // handle --cap-add=all - if utils.StringsContainsNoCase(adds, "all") { + if stringutils.InSlice(adds, "all") { basics = allCaps } - if !utils.StringsContainsNoCase(drops, "all") { + if !stringutils.InSlice(drops, "all") { for _, cap := range basics { // skip `all` aready handled above if strings.ToLower(cap) == "all" { @@ -107,7 +107,7 @@ func TweakCapabilities(basics, adds, drops []string) ([]string, error) { } // if we don't drop `all`, add back all the non-dropped caps - if !utils.StringsContainsNoCase(drops, cap) { + if !stringutils.InSlice(drops, cap) { newCaps = append(newCaps, strings.ToUpper(cap)) } } @@ -119,12 +119,12 @@ func TweakCapabilities(basics, adds, drops []string) ([]string, error) { continue } - if !utils.StringsContainsNoCase(allCaps, cap) { + if !stringutils.InSlice(allCaps, cap) { return nil, fmt.Errorf("Unknown capability to add: %q", cap) } // add cap if not already in the list - if !utils.StringsContainsNoCase(newCaps, cap) { + if !stringutils.InSlice(newCaps, cap) { newCaps = append(newCaps, strings.ToUpper(cap)) } } diff --git a/daemon/info.go b/daemon/info.go index 183a9e68bb..a77e3efd16 100644 --- a/daemon/info.go +++ b/daemon/info.go @@ -8,6 +8,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/engine" + "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/parsers/operatingsystem" "github.com/docker/docker/pkg/system" @@ -61,7 +62,7 @@ func (daemon *Daemon) CmdInfo(job *engine.Job) error { v.SetBool("SwapLimit", daemon.SystemConfig().SwapLimit) v.SetBool("IPv4Forwarding", !daemon.SystemConfig().IPv4ForwardingDisabled) v.SetBool("Debug", os.Getenv("DEBUG") != "") - v.SetInt("NFd", utils.GetTotalUsedFds()) + v.SetInt("NFd", fileutils.GetTotalUsedFds()) v.SetInt("NGoroutines", runtime.NumGoroutine()) v.Set("SystemTime", time.Now().Format(time.RFC3339Nano)) v.Set("ExecutionDriver", daemon.ExecutionDriver().Name()) diff --git a/daemon/utils_test.go b/daemon/utils_test.go index ff5b082ba5..aabbeaf6fa 100644 --- a/daemon/utils_test.go +++ b/daemon/utils_test.go @@ -4,12 +4,11 @@ import ( "testing" "github.com/docker/docker/runconfig" - "github.com/docker/docker/utils" ) func TestMergeLxcConfig(t *testing.T) { hostConfig := &runconfig.HostConfig{ - LxcConf: []utils.KeyValuePair{ + LxcConf: []runconfig.KeyValuePair{ {Key: "lxc.cgroups.cpuset", Value: "1,2"}, }, } diff --git a/docker/docker.go b/docker/docker.go index c9b2c77b02..cf5b715598 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -15,7 +15,6 @@ import ( flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/pkg/term" - "github.com/docker/docker/utils" ) const ( @@ -136,7 +135,7 @@ func main() { } if err := cli.Cmd(flag.Args()...); err != nil { - if sterr, ok := err.(*utils.StatusError); ok { + if sterr, ok := err.(*client.StatusError); ok { if sterr.Status != "" { logrus.Println(sterr.Status) } diff --git a/engine/env.go b/engine/env.go index 089bc162c0..107ae4a0d9 100644 --- a/engine/env.go +++ b/engine/env.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/docker/docker/utils" + "github.com/docker/docker/pkg/ioutils" ) type Env []string @@ -258,7 +258,7 @@ func (env *Env) Encode(dst io.Writer) error { } func (env *Env) WriteTo(dst io.Writer) (int64, error) { - wc := utils.NewWriteCounter(dst) + wc := ioutils.NewWriteCounter(dst) err := env.Encode(wc) return wc.Count, err } diff --git a/graph/graph.go b/graph/graph.go index 087a6f093c..5159a93223 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -25,7 +25,6 @@ import ( "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/truncindex" "github.com/docker/docker/runconfig" - "github.com/docker/docker/utils" ) // A Graph is a store for versioned filesystem images and the relationship between them. @@ -154,7 +153,7 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) graph.driver.Remove(img.ID) } }() - if err := utils.ValidateID(img.ID); err != nil { + if err := image.ValidateID(img.ID); err != nil { return err } // (This is a convenience to save time. Race conditions are taken care of by os.Rename) diff --git a/graph/import.go b/graph/import.go index eb63af0b60..0ba03d0f53 100644 --- a/graph/import.go +++ b/graph/import.go @@ -9,6 +9,7 @@ import ( "github.com/docker/docker/engine" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/progressreader" "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/runconfig" @@ -46,7 +47,7 @@ func (s *TagStore) CmdImport(job *engine.Job) error { u.Path = "" } job.Stdout.Write(sf.FormatStatus("", "Downloading from %s", u)) - resp, err = utils.Download(u.String()) + resp, err = httputils.Download(u.String()) if err != nil { return err } diff --git a/graph/load.go b/graph/load.go index ace222e3fe..bf2cc6d700 100644 --- a/graph/load.go +++ b/graph/load.go @@ -13,7 +13,6 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" - "github.com/docker/docker/utils" ) // Loads a set of images into the repository. This is the complementary of ImageExport. @@ -100,7 +99,7 @@ func (s *TagStore) recursiveLoad(eng *engine.Engine, address, tmpImageDir string logrus.Debugf("Error unmarshalling json", err) return err } - if err := utils.ValidateID(img.ID); err != nil { + if err := image.ValidateID(img.ID); err != nil { logrus.Debugf("Error validating ID: %s", err) return err } diff --git a/image/image.go b/image/image.go index a661e3ce70..90714d6dbe 100644 --- a/image/image.go +++ b/image/image.go @@ -6,12 +6,12 @@ import ( "io/ioutil" "os" "path" + "regexp" "strconv" "time" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/runconfig" - "github.com/docker/docker/utils" ) // Set the max depth to the aufs default that most @@ -51,7 +51,7 @@ func LoadImage(root string) (*Image, error) { if err := dec.Decode(img); err != nil { return nil, err } - if err := utils.ValidateID(img.ID); err != nil { + if err := ValidateID(img.ID); err != nil { return nil, err } @@ -263,3 +263,13 @@ func NewImgJSON(src []byte) (*Image, error) { } return ret, nil } + +// Check wheather id is a valid image ID or not +func ValidateID(id string) error { + validHex := regexp.MustCompile(`^([a-f0-9]{64})$`) + if ok := validHex.MatchString(id); !ok { + err := fmt.Errorf("image ID '%s' is invalid", id) + return err + } + return nil +} diff --git a/integration/graph_test.go b/integration/graph_test.go index a481154551..8b6d7626f6 100644 --- a/integration/graph_test.go +++ b/integration/graph_test.go @@ -15,7 +15,6 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/stringid" - "github.com/docker/docker/utils" ) func TestMount(t *testing.T) { @@ -103,7 +102,7 @@ func TestGraphCreate(t *testing.T) { if err != nil { t.Fatal(err) } - if err := utils.ValidateID(img.ID); err != nil { + if err := image.ValidateID(img.ID); err != nil { t.Fatal(err) } if img.Comment != "Testing" { diff --git a/integration/runtime_test.go b/integration/runtime_test.go index b5e404d59d..6881fbea60 100644 --- a/integration/runtime_test.go +++ b/integration/runtime_test.go @@ -23,6 +23,7 @@ import ( "github.com/docker/docker/graph" "github.com/docker/docker/image" "github.com/docker/docker/nat" + "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/pkg/stringid" @@ -121,7 +122,7 @@ func init() { spawnGlobalDaemon() spawnLegitHttpsDaemon() spawnRogueHttpsDaemon() - startFds, startGoroutines = utils.GetTotalUsedFds(), runtime.NumGoroutine() + startFds, startGoroutines = fileutils.GetTotalUsedFds(), runtime.NumGoroutine() } func setupBaseImage() { diff --git a/integration/z_final_test.go b/integration/z_final_test.go index 13cd0c3fd4..d6ef2884f2 100644 --- a/integration/z_final_test.go +++ b/integration/z_final_test.go @@ -1,13 +1,14 @@ package docker import ( - "github.com/docker/docker/utils" "runtime" "testing" + + "github.com/docker/docker/pkg/fileutils" ) func displayFdGoroutines(t *testing.T) { - t.Logf("File Descriptors: %d, Goroutines: %d", utils.GetTotalUsedFds(), runtime.NumGoroutine()) + t.Logf("File Descriptors: %d, Goroutines: %d", fileutils.GetTotalUsedFds(), runtime.NumGoroutine()) } func TestFinal(t *testing.T) { diff --git a/pkg/fileutils/fileutils.go b/pkg/fileutils/fileutils.go index 4325297651..ef2a6523dc 100644 --- a/pkg/fileutils/fileutils.go +++ b/pkg/fileutils/fileutils.go @@ -1,6 +1,10 @@ package fileutils import ( + "fmt" + "io" + "io/ioutil" + "os" "path/filepath" "github.com/Sirupsen/logrus" @@ -25,3 +29,53 @@ func Matches(relFilePath string, patterns []string) (bool, error) { } return false, nil } + +func CopyFile(src, dst string) (int64, error) { + if src == dst { + return 0, nil + } + sf, err := os.Open(src) + if err != nil { + return 0, err + } + defer sf.Close() + if err := os.Remove(dst); err != nil && !os.IsNotExist(err) { + return 0, err + } + df, err := os.Create(dst) + if err != nil { + return 0, err + } + defer df.Close() + return io.Copy(df, sf) +} + +func GetTotalUsedFds() int { + if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil { + logrus.Errorf("Error opening /proc/%d/fd: %s", os.Getpid(), err) + } else { + return len(fds) + } + return -1 +} + +// ReadSymlinkedDirectory returns the target directory of a symlink. +// The target of the symbolic link may not be a file. +func ReadSymlinkedDirectory(path string) (string, error) { + var realPath string + var err error + if realPath, err = filepath.Abs(path); err != nil { + return "", fmt.Errorf("unable to get absolute path for %s: %s", path, err) + } + if realPath, err = filepath.EvalSymlinks(realPath); err != nil { + return "", fmt.Errorf("failed to canonicalise path for %s: %s", path, err) + } + realPathInfo, err := os.Stat(realPath) + if err != nil { + return "", fmt.Errorf("failed to stat target '%s' of '%s': %s", realPath, path, err) + } + if !realPathInfo.Mode().IsDir() { + return "", fmt.Errorf("canonical path points to a file '%s'", realPath) + } + return realPath, nil +} diff --git a/pkg/fileutils/fileutils_test.go b/pkg/fileutils/fileutils_test.go new file mode 100644 index 0000000000..16d00d7b95 --- /dev/null +++ b/pkg/fileutils/fileutils_test.go @@ -0,0 +1,81 @@ +package fileutils + +import ( + "os" + "testing" +) + +// Reading a symlink to a directory must return the directory +func TestReadSymlinkedDirectoryExistingDirectory(t *testing.T) { + var err error + if err = os.Mkdir("/tmp/testReadSymlinkToExistingDirectory", 0777); err != nil { + t.Errorf("failed to create directory: %s", err) + } + + if err = os.Symlink("/tmp/testReadSymlinkToExistingDirectory", "/tmp/dirLinkTest"); err != nil { + t.Errorf("failed to create symlink: %s", err) + } + + var path string + if path, err = ReadSymlinkedDirectory("/tmp/dirLinkTest"); err != nil { + t.Fatalf("failed to read symlink to directory: %s", err) + } + + if path != "/tmp/testReadSymlinkToExistingDirectory" { + t.Fatalf("symlink returned unexpected directory: %s", path) + } + + if err = os.Remove("/tmp/testReadSymlinkToExistingDirectory"); err != nil { + t.Errorf("failed to remove temporary directory: %s", err) + } + + if err = os.Remove("/tmp/dirLinkTest"); err != nil { + t.Errorf("failed to remove symlink: %s", err) + } +} + +// Reading a non-existing symlink must fail +func TestReadSymlinkedDirectoryNonExistingSymlink(t *testing.T) { + var path string + var err error + if path, err = ReadSymlinkedDirectory("/tmp/test/foo/Non/ExistingPath"); err == nil { + t.Fatalf("error expected for non-existing symlink") + } + + if path != "" { + t.Fatalf("expected empty path, but '%s' was returned", path) + } +} + +// Reading a symlink to a file must fail +func TestReadSymlinkedDirectoryToFile(t *testing.T) { + var err error + var file *os.File + + if file, err = os.Create("/tmp/testReadSymlinkToFile"); err != nil { + t.Fatalf("failed to create file: %s", err) + } + + file.Close() + + if err = os.Symlink("/tmp/testReadSymlinkToFile", "/tmp/fileLinkTest"); err != nil { + t.Errorf("failed to create symlink: %s", err) + } + + var path string + if path, err = ReadSymlinkedDirectory("/tmp/fileLinkTest"); err == nil { + t.Fatalf("ReadSymlinkedDirectory on a symlink to a file should've failed") + } + + if path != "" { + t.Fatalf("path should've been empty: %s", path) + } + + if err = os.Remove("/tmp/testReadSymlinkToFile"); err != nil { + t.Errorf("failed to remove file: %s", err) + } + + if err = os.Remove("/tmp/fileLinkTest"); err != nil { + t.Errorf("failed to remove symlink: %s", err) + } +} diff --git a/pkg/httputils/httputils.go b/pkg/httputils/httputils.go new file mode 100644 index 0000000000..1c922240e6 --- /dev/null +++ b/pkg/httputils/httputils.go @@ -0,0 +1,26 @@ +package httputils + +import ( + "fmt" + "net/http" + + "github.com/docker/docker/pkg/jsonmessage" +) + +// Request a given URL and return an io.Reader +func Download(url string) (resp *http.Response, err error) { + if resp, err = http.Get(url); err != nil { + return nil, err + } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("Got HTTP status code >= 400: %s", resp.Status) + } + return resp, nil +} + +func NewHTTPRequestError(msg string, res *http.Response) error { + return &jsonmessage.JSONError{ + Message: msg, + Code: res.StatusCode, + } +} diff --git a/pkg/ioutils/readers.go b/pkg/ioutils/readers.go index 58ff1af639..0e542cbad3 100644 --- a/pkg/ioutils/readers.go +++ b/pkg/ioutils/readers.go @@ -3,6 +3,8 @@ package ioutils import ( "bytes" "crypto/rand" + "crypto/sha256" + "encoding/hex" "io" "math/big" "sync" @@ -215,3 +217,11 @@ func (r *bufReader) Close() error { } return closer.Close() } + +func HashData(src io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, src); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/pkg/ioutils/writers.go b/pkg/ioutils/writers.go index c0b3608fe6..43fdc44ea9 100644 --- a/pkg/ioutils/writers.go +++ b/pkg/ioutils/writers.go @@ -37,3 +37,24 @@ func NewWriteCloserWrapper(r io.Writer, closer func() error) io.WriteCloser { closer: closer, } } + +// Wrap a concrete io.Writer and hold a count of the number +// of bytes written to the writer during a "session". +// This can be convenient when write return is masked +// (e.g., json.Encoder.Encode()) +type WriteCounter struct { + Count int64 + Writer io.Writer +} + +func NewWriteCounter(w io.Writer) *WriteCounter { + return &WriteCounter{ + Writer: w, + } +} + +func (wc *WriteCounter) Write(p []byte) (count int, err error) { + count, err = wc.Writer.Write(p) + wc.Count += int64(count) + return +} diff --git a/pkg/ioutils/writers_test.go b/pkg/ioutils/writers_test.go new file mode 100644 index 0000000000..80d7f7f795 --- /dev/null +++ b/pkg/ioutils/writers_test.go @@ -0,0 +1,41 @@ +package ioutils + +import ( + "bytes" + "strings" + "testing" +) + +func TestNopWriter(t *testing.T) { + nw := &NopWriter{} + l, err := nw.Write([]byte{'c'}) + if err != nil { + t.Fatal(err) + } + if l != 1 { + t.Fatalf("Expected 1 got %d", l) + } +} + +func TestWriteCounter(t *testing.T) { + dummy1 := "This is a dummy string." + dummy2 := "This is another dummy string." + totalLength := int64(len(dummy1) + len(dummy2)) + + reader1 := strings.NewReader(dummy1) + reader2 := strings.NewReader(dummy2) + + var buffer bytes.Buffer + wc := NewWriteCounter(&buffer) + + reader1.WriteTo(wc) + reader2.WriteTo(wc) + + if wc.Count != totalLength { + t.Errorf("Wrong count: %d vs. %d", wc.Count, totalLength) + } + + if buffer.String() != dummy1+dummy2 { + t.Error("Wrong message written") + } +} diff --git a/pkg/requestdecorator/requestdecorator_test.go b/pkg/requestdecorator/requestdecorator_test.go index b2c1fb3b97..f1f9ef756b 100644 --- a/pkg/requestdecorator/requestdecorator_test.go +++ b/pkg/requestdecorator/requestdecorator_test.go @@ -180,8 +180,8 @@ func TestRequestFactory(t *testing.T) { requestFactory := NewRequestFactory(ad, uad) - if dlen := len(requestFactory.GetDecorators()); dlen != 2 { - t.Fatalf("Expected to have two decorators, got %d", dlen) + if l := len(requestFactory.GetDecorators()); l != 2 { + t.Fatalf("Expected to have two decorators, got %d", l) } req, err := requestFactory.NewRequest("GET", "/test", strings.NewReader("test")) @@ -209,8 +209,8 @@ func TestRequestFactoryNewRequestWithDecorators(t *testing.T) { requestFactory := NewRequestFactory(ad) - if dlen := len(requestFactory.GetDecorators()); dlen != 1 { - t.Fatalf("Expected to have one decorators, got %d", dlen) + if l := len(requestFactory.GetDecorators()); l != 1 { + t.Fatalf("Expected to have one decorators, got %d", l) } ad2 := NewAuthDecorator("test2", "password2") @@ -235,15 +235,15 @@ func TestRequestFactoryNewRequestWithDecorators(t *testing.T) { func TestRequestFactoryAddDecorator(t *testing.T) { requestFactory := NewRequestFactory() - if dlen := len(requestFactory.GetDecorators()); dlen != 0 { - t.Fatalf("Expected to have zero decorators, got %d", dlen) + if l := len(requestFactory.GetDecorators()); l != 0 { + t.Fatalf("Expected to have zero decorators, got %d", l) } ad := NewAuthDecorator("test", "password") requestFactory.AddDecorator(ad) - if dlen := len(requestFactory.GetDecorators()); dlen != 1 { - t.Fatalf("Expected to have one decorators, got %d", dlen) + if l := len(requestFactory.GetDecorators()); l != 1 { + t.Fatalf("Expected to have one decorators, got %d", l) } } diff --git a/pkg/resolvconf/resolvconf.go b/pkg/resolvconf/resolvconf.go index d7d53e16d0..5707b16b7f 100644 --- a/pkg/resolvconf/resolvconf.go +++ b/pkg/resolvconf/resolvconf.go @@ -9,7 +9,7 @@ import ( "sync" "github.com/Sirupsen/logrus" - "github.com/docker/docker/utils" + "github.com/docker/docker/pkg/ioutils" ) var ( @@ -59,7 +59,7 @@ func GetIfChanged() ([]byte, string, error) { if err != nil { return nil, "", err } - newHash, err := utils.HashData(bytes.NewReader(resolv)) + newHash, err := ioutils.HashData(bytes.NewReader(resolv)) if err != nil { return nil, "", err } diff --git a/pkg/stringutils/stringutils.go b/pkg/stringutils/stringutils.go index f5f07dd18e..e3ebf5d1ed 100644 --- a/pkg/stringutils/stringutils.go +++ b/pkg/stringutils/stringutils.go @@ -1,7 +1,9 @@ package stringutils import ( + "bytes" mathrand "math/rand" + "strings" "time" ) @@ -28,3 +30,57 @@ func GenerateRandomAsciiString(n int) string { } return string(res) } + +// Truncate a string to maxlen +func Truncate(s string, maxlen int) string { + if len(s) <= maxlen { + return s + } + return s[:maxlen] +} + +// Test wheather a string is contained in a slice of strings or not. +// Comparison is case insensitive +func InSlice(slice []string, s string) bool { + for _, ss := range slice { + if strings.ToLower(s) == strings.ToLower(ss) { + return true + } + } + return false +} + +func quote(word string, buf *bytes.Buffer) { + // Bail out early for "simple" strings + if word != "" && !strings.ContainsAny(word, "\\'\"`${[|&;<>()~*?! \t\n") { + buf.WriteString(word) + return + } + + buf.WriteString("'") + + for i := 0; i < len(word); i++ { + b := word[i] + if b == '\'' { + // Replace literal ' with a close ', a \', and a open ' + buf.WriteString("'\\''") + } else { + buf.WriteByte(b) + } + } + + buf.WriteString("'") +} + +// Take a list of strings and escape them so they will be handled right +// when passed as arguments to an program via a shell +func ShellQuoteArguments(args []string) string { + var buf bytes.Buffer + for i, arg := range args { + if i != 0 { + buf.WriteByte(' ') + } + quote(arg, &buf) + } + return buf.String() +} diff --git a/pkg/stringutils/stringutils_test.go b/pkg/stringutils/stringutils_test.go index a5a01b4a03..8dcb4696bb 100644 --- a/pkg/stringutils/stringutils_test.go +++ b/pkg/stringutils/stringutils_test.go @@ -56,3 +56,32 @@ func TestGenerateRandomAsciiStringIsAscii(t *testing.T) { t.Fatalf("%s contained non-ascii characters", str) } } + +func TestTruncate(t *testing.T) { + str := "teststring" + newstr := Truncate(str, 4) + if newstr != "test" { + t.Fatalf("Expected test, got %s", newstr) + } + newstr = Truncate(str, 20) + if newstr != "teststring" { + t.Fatalf("Expected teststring, got %s", newstr) + } +} + +func TestInSlice(t *testing.T) { + slice := []string{"test", "in", "slice"} + + test := InSlice(slice, "test") + if !test { + t.Fatalf("Expected string test to be in slice") + } + test = InSlice(slice, "SLICE") + if !test { + t.Fatalf("Expected string SLICE to be in slice") + } + test = InSlice(slice, "notinslice") + if test { + t.Fatalf("Expected string notinslice not to be in slice") + } +} diff --git a/registry/config.go b/registry/config.go index 3515836d18..a0a978cc72 100644 --- a/registry/config.go +++ b/registry/config.go @@ -9,9 +9,9 @@ import ( "regexp" "strings" + "github.com/docker/docker/image" "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/utils" ) // Options holds command line options. @@ -213,7 +213,7 @@ func validateRemoteName(remoteName string) error { name = nameParts[0] // the repository name must not be a valid image ID - if err := utils.ValidateID(name); err == nil { + if err := image.ValidateID(name); err == nil { return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) } } else { diff --git a/registry/session.go b/registry/session.go index 4682a5074c..c62745b5bc 100644 --- a/registry/session.go +++ b/registry/session.go @@ -21,7 +21,6 @@ import ( "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/requestdecorator" "github.com/docker/docker/pkg/tarsum" - "github.com/docker/docker/utils" ) type Session struct { @@ -86,7 +85,7 @@ func (r *Session) GetRemoteHistory(imgID, registry string, token []string) ([]st if res.StatusCode == 401 { return nil, errLoginRequired } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } jsonString, err := ioutil.ReadAll(res.Body) @@ -115,7 +114,7 @@ func (r *Session) LookupRemoteImage(imgID, registry string, token []string) erro } res.Body.Close() if res.StatusCode != 200 { - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } return nil } @@ -134,7 +133,7 @@ func (r *Session) GetRemoteImageJSON(imgID, registry string, token []string) ([] } defer res.Body.Close() if res.StatusCode != 200 { - return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) + return nil, -1, httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } // if the size header is not present, then set it to '-1' imageSize := -1 @@ -282,13 +281,13 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. if res.StatusCode == 404 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) } else if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to pull repository %s: %q", res.StatusCode, remote, errBody), res) } var tokens []string @@ -379,12 +378,12 @@ func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regist } defer res.Body.Close() if res.StatusCode == 401 && strings.HasPrefix(registry, "http://") { - return utils.NewHTTPRequestError("HTTP code 401, Docker will not send auth headers over HTTP.", res) + return httputils.NewHTTPRequestError("HTTP code 401, Docker will not send auth headers over HTTP.", res) } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { @@ -392,7 +391,7 @@ func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regist } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %q", res.StatusCode, errBody), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %q", res.StatusCode, errBody), res) } return nil } @@ -432,9 +431,9 @@ func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return "", "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + return "", "", httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return "", "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %q", res.StatusCode, errBody), res) + return "", "", httputils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %q", res.StatusCode, errBody), res) } checksumPayload = "sha256:" + hex.EncodeToString(h.Sum(nil)) @@ -461,7 +460,7 @@ func (r *Session) PushRegistryTag(remote, revision, tag, registry string, token } res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 201 { - return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) } return nil } @@ -523,7 +522,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %q", res.StatusCode, remote, errBody), res) } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] @@ -547,7 +546,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if err != nil { logrus.Debugf("Error reading response body: %s", err) } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote, errBody), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %q", res.StatusCode, remote, errBody), res) } } @@ -595,7 +594,7 @@ func (r *Session) SearchRepositories(term string) (*SearchResults, error) { } defer res.Body.Close() if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexpected status code %d", res.StatusCode), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Unexpected status code %d", res.StatusCode), res) } result := new(SearchResults) err = json.NewDecoder(res.Body).Decode(result) diff --git a/registry/session_v2.go b/registry/session_v2.go index fb1d18e8e7..a14e434acf 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -12,7 +12,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/digest" "github.com/docker/distribution/registry/api/v2" - "github.com/docker/docker/utils" + "github.com/docker/docker/pkg/httputils" ) const DockerDigestHeader = "Docker-Content-Digest" @@ -95,7 +95,7 @@ func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, au } else if res.StatusCode == 404 { return nil, "", ErrDoesNotExist } - return nil, "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) + return nil, "", httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) } manifestBytes, err := ioutil.ReadAll(res.Body) @@ -141,7 +141,7 @@ func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Di return false, nil } - return false, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s", res.StatusCode, imageName, dgst), res) + return false, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s", res.StatusCode, imageName, dgst), res) } func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Digest, blobWrtr io.Writer, auth *RequestAuthorization) error { @@ -168,7 +168,7 @@ func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Dig if res.StatusCode == 401 { return errLoginRequired } - return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) } _, err = io.Copy(blobWrtr, res.Body) @@ -198,7 +198,7 @@ func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName string, dgst dige if res.StatusCode == 401 { return nil, 0, errLoginRequired } - return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob - %s", res.StatusCode, imageName, dgst), res) + return nil, 0, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob - %s", res.StatusCode, imageName, dgst), res) } lenStr := res.Header.Get("Content-Length") l, err := strconv.ParseInt(lenStr, 10, 64) @@ -245,7 +245,7 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Dig return err } logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s", res.StatusCode, imageName, dgst), res) + return httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s", res.StatusCode, imageName, dgst), res) } return nil @@ -286,7 +286,7 @@ func (r *Session) initiateBlobUpload(ep *Endpoint, imageName string, auth *Reque } logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: unexpected %d response status trying to initiate upload of %s", res.StatusCode, imageName), res) + return "", httputils.NewHTTPRequestError(fmt.Sprintf("Server error: unexpected %d response status trying to initiate upload of %s", res.StatusCode, imageName), res) } if location = res.Header.Get("Location"); location == "" { @@ -328,7 +328,7 @@ func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, si return "", err } logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) + return "", httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } hdrDigest, err := digest.ParseDigest(res.Header.Get(DockerDigestHeader)) @@ -384,7 +384,7 @@ func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestA } else if res.StatusCode == 404 { return nil, ErrDoesNotExist } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res) + return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res) } decoder := json.NewDecoder(res.Body) diff --git a/runconfig/hostconfig.go b/runconfig/hostconfig.go index 84d636b5c4..9d4eb26414 100644 --- a/runconfig/hostconfig.go +++ b/runconfig/hostconfig.go @@ -6,9 +6,13 @@ import ( "github.com/docker/docker/engine" "github.com/docker/docker/nat" "github.com/docker/docker/pkg/ulimit" - "github.com/docker/docker/utils" ) +type KeyValuePair struct { + Key string + Value string +} + type NetworkMode string // IsPrivate indicates whether container use it's private network stack @@ -107,7 +111,7 @@ type LogConfig struct { type HostConfig struct { Binds []string ContainerIDFile string - LxcConf []utils.KeyValuePair + LxcConf []KeyValuePair Memory int64 // Memory limit (in bytes) MemorySwap int64 // Total memory usage (memory + swap); set `-1` to disable swap CpuShares int64 // CPU shares (relative weight vs. other containers) diff --git a/runconfig/parse.go b/runconfig/parse.go index 1fb36e4ace..d302330c82 100644 --- a/runconfig/parse.go +++ b/runconfig/parse.go @@ -12,7 +12,6 @@ import ( "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/ulimit" "github.com/docker/docker/pkg/units" - "github.com/docker/docker/utils" ) var ( @@ -430,14 +429,14 @@ func parseDriverOpts(opts opts.ListOpts) (map[string][]string, error) { return out, nil } -func parseKeyValueOpts(opts opts.ListOpts) ([]utils.KeyValuePair, error) { - out := make([]utils.KeyValuePair, opts.Len()) +func parseKeyValueOpts(opts opts.ListOpts) ([]KeyValuePair, error) { + out := make([]KeyValuePair, opts.Len()) for i, o := range opts.GetAll() { k, v, err := parsers.ParseKeyValueOpt(o) if err != nil { return nil, err } - out[i] = utils.KeyValuePair{Key: k, Value: v} + out[i] = KeyValuePair{Key: k, Value: v} } return out, nil } diff --git a/utils/utils.go b/utils/utils.go index 92ecb9b6c2..a151fc3f0e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -2,9 +2,7 @@ package utils import ( "bufio" - "bytes" "crypto/sha1" - "crypto/sha256" "encoding/hex" "fmt" "io" @@ -13,47 +11,17 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "runtime" "strings" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/stringid" ) -type KeyValuePair struct { - Key string - Value string -} - -var ( - validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) -) - -// Request a given URL and return an io.Reader -func Download(url string) (resp *http.Response, err error) { - if resp, err = http.Get(url); err != nil { - return nil, err - } - if resp.StatusCode >= 400 { - return nil, fmt.Errorf("Got HTTP status code >= 400: %s", resp.Status) - } - return resp, nil -} - -func Trunc(s string, maxlen int) string { - if len(s) <= maxlen { - return s - } - return s[:maxlen] -} - // Figure out the absolute path of our own binary (if it's still around). func SelfPath() string { path, err := exec.LookPath(os.Args[0]) @@ -155,74 +123,7 @@ func DockerInitPath(localCopy string) string { return "" } -func GetTotalUsedFds() int { - if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil { - logrus.Errorf("Error opening /proc/%d/fd: %s", os.Getpid(), err) - } else { - return len(fds) - } - return -1 -} - -func ValidateID(id string) error { - if ok := validHex.MatchString(id); !ok { - err := fmt.Errorf("image ID '%s' is invalid", id) - return err - } - return nil -} - -// Code c/c from io.Copy() modified to handle escape sequence -func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) { - buf := make([]byte, 32*1024) - for { - nr, er := src.Read(buf) - if nr > 0 { - // ---- Docker addition - // char 16 is C-p - if nr == 1 && buf[0] == 16 { - nr, er = src.Read(buf) - // char 17 is C-q - if nr == 1 && buf[0] == 17 { - if err := src.Close(); err != nil { - return 0, err - } - return 0, nil - } - } - // ---- End of docker - nw, ew := dst.Write(buf[0:nr]) - if nw > 0 { - written += int64(nw) - } - if ew != nil { - err = ew - break - } - if nr != nw { - err = io.ErrShortWrite - break - } - } - if er == io.EOF { - break - } - if er != nil { - err = er - break - } - } - return written, err -} - -func HashData(src io.Reader) (string, error) { - h := sha256.New() - if _, err := io.Copy(h, src); err != nil { - return "", err - } - return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil -} - +// FIXME: move to httputils? ioutils? type WriteFlusher struct { sync.Mutex w io.Writer @@ -254,58 +155,6 @@ func NewWriteFlusher(w io.Writer) *WriteFlusher { return &WriteFlusher{w: w, flusher: flusher} } -func NewHTTPRequestError(msg string, res *http.Response) error { - return &jsonmessage.JSONError{ - Message: msg, - Code: res.StatusCode, - } -} - -// An StatusError reports an unsuccessful exit by a command. -type StatusError struct { - Status string - StatusCode int -} - -func (e *StatusError) Error() string { - return fmt.Sprintf("Status: %s, Code: %d", e.Status, e.StatusCode) -} - -func quote(word string, buf *bytes.Buffer) { - // Bail out early for "simple" strings - if word != "" && !strings.ContainsAny(word, "\\'\"`${[|&;<>()~*?! \t\n") { - buf.WriteString(word) - return - } - - buf.WriteString("'") - - for i := 0; i < len(word); i++ { - b := word[i] - if b == '\'' { - // Replace literal ' with a close ', a \', and a open ' - buf.WriteString("'\\''") - } else { - buf.WriteByte(b) - } - } - - buf.WriteString("'") -} - -// Take a list of strings and escape them so they will be handled right -// when passed as arguments to an program via a shell -func ShellQuoteArguments(args []string) string { - var buf bytes.Buffer - for i, arg := range args { - if i != 0 { - buf.WriteByte(' ') - } - quote(arg, &buf) - } - return buf.String() -} - var globalTestID string // TestDirectory creates a new temporary directory and returns its path. @@ -343,26 +192,6 @@ func GetCallerName(depth int) string { return callerShortName } -func CopyFile(src, dst string) (int64, error) { - if src == dst { - return 0, nil - } - sf, err := os.Open(src) - if err != nil { - return 0, err - } - defer sf.Close() - if err := os.Remove(dst); err != nil && !os.IsNotExist(err) { - return 0, err - } - df, err := os.Create(dst) - if err != nil { - return 0, err - } - defer df.Close() - return io.Copy(df, sf) -} - // ReplaceOrAppendValues returns the defaults with the overrides either // replaced by env key or appended to the list func ReplaceOrAppendEnvValues(defaults, overrides []string) []string { @@ -411,27 +240,6 @@ func DoesEnvExist(name string) bool { return false } -// ReadSymlinkedDirectory returns the target directory of a symlink. -// The target of the symbolic link may not be a file. -func ReadSymlinkedDirectory(path string) (string, error) { - var realPath string - var err error - if realPath, err = filepath.Abs(path); err != nil { - return "", fmt.Errorf("unable to get absolute path for %s: %s", path, err) - } - if realPath, err = filepath.EvalSymlinks(realPath); err != nil { - return "", fmt.Errorf("failed to canonicalise path for %s: %s", path, err) - } - realPathInfo, err := os.Stat(realPath) - if err != nil { - return "", fmt.Errorf("failed to stat target '%s' of '%s': %s", realPath, path, err) - } - if !realPathInfo.Mode().IsDir() { - return "", fmt.Errorf("canonical path points to a file '%s'", realPath) - } - return realPath, nil -} - // ValidateContextDirectory checks if all the contents of the directory // can be read and returns an error if some files can't be read // symlinks which point to non-existing files don't trigger an error @@ -476,15 +284,6 @@ func ValidateContextDirectory(srcPath string, excludes []string) error { }) } -func StringsContainsNoCase(slice []string, s string) bool { - for _, ss := range slice { - if strings.ToLower(s) == strings.ToLower(ss) { - return true - } - } - return false -} - // Reads a .dockerignore file and returns the list of file patterns // to ignore. Note this will trim whitespace from each line as well // as use GO's "clean" func to get the shortest/cleanest path for each. @@ -516,27 +315,6 @@ func ReadDockerIgnore(path string) ([]string, error) { return excludes, nil } -// Wrap a concrete io.Writer and hold a count of the number -// of bytes written to the writer during a "session". -// This can be convenient when write return is masked -// (e.g., json.Encoder.Encode()) -type WriteCounter struct { - Count int64 - Writer io.Writer -} - -func NewWriteCounter(w io.Writer) *WriteCounter { - return &WriteCounter{ - Writer: w, - } -} - -func (wc *WriteCounter) Write(p []byte) (count int, err error) { - count, err = wc.Writer.Write(p) - wc.Count += int64(count) - return -} - // ImageReference combines `repo` and `ref` and returns a string representing // the combination. If `ref` is a digest (meaning it's of the form // :, the returned string is @. Otherwise, diff --git a/utils/utils_test.go b/utils/utils_test.go index 94303a0e96..2863009423 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -1,9 +1,10 @@ package utils import ( - "bytes" + "fmt" + "io/ioutil" "os" - "strings" + "path/filepath" "testing" ) @@ -25,104 +26,6 @@ func TestReplaceAndAppendEnvVars(t *testing.T) { } } -// Reading a symlink to a directory must return the directory -func TestReadSymlinkedDirectoryExistingDirectory(t *testing.T) { - var err error - if err = os.Mkdir("/tmp/testReadSymlinkToExistingDirectory", 0777); err != nil { - t.Errorf("failed to create directory: %s", err) - } - - if err = os.Symlink("/tmp/testReadSymlinkToExistingDirectory", "/tmp/dirLinkTest"); err != nil { - t.Errorf("failed to create symlink: %s", err) - } - - var path string - if path, err = ReadSymlinkedDirectory("/tmp/dirLinkTest"); err != nil { - t.Fatalf("failed to read symlink to directory: %s", err) - } - - if path != "/tmp/testReadSymlinkToExistingDirectory" { - t.Fatalf("symlink returned unexpected directory: %s", path) - } - - if err = os.Remove("/tmp/testReadSymlinkToExistingDirectory"); err != nil { - t.Errorf("failed to remove temporary directory: %s", err) - } - - if err = os.Remove("/tmp/dirLinkTest"); err != nil { - t.Errorf("failed to remove symlink: %s", err) - } -} - -// Reading a non-existing symlink must fail -func TestReadSymlinkedDirectoryNonExistingSymlink(t *testing.T) { - var path string - var err error - if path, err = ReadSymlinkedDirectory("/tmp/test/foo/Non/ExistingPath"); err == nil { - t.Fatalf("error expected for non-existing symlink") - } - - if path != "" { - t.Fatalf("expected empty path, but '%s' was returned", path) - } -} - -// Reading a symlink to a file must fail -func TestReadSymlinkedDirectoryToFile(t *testing.T) { - var err error - var file *os.File - - if file, err = os.Create("/tmp/testReadSymlinkToFile"); err != nil { - t.Fatalf("failed to create file: %s", err) - } - - file.Close() - - if err = os.Symlink("/tmp/testReadSymlinkToFile", "/tmp/fileLinkTest"); err != nil { - t.Errorf("failed to create symlink: %s", err) - } - - var path string - if path, err = ReadSymlinkedDirectory("/tmp/fileLinkTest"); err == nil { - t.Fatalf("ReadSymlinkedDirectory on a symlink to a file should've failed") - } - - if path != "" { - t.Fatalf("path should've been empty: %s", path) - } - - if err = os.Remove("/tmp/testReadSymlinkToFile"); err != nil { - t.Errorf("failed to remove file: %s", err) - } - - if err = os.Remove("/tmp/fileLinkTest"); err != nil { - t.Errorf("failed to remove symlink: %s", err) - } -} - -func TestWriteCounter(t *testing.T) { - dummy1 := "This is a dummy string." - dummy2 := "This is another dummy string." - totalLength := int64(len(dummy1) + len(dummy2)) - - reader1 := strings.NewReader(dummy1) - reader2 := strings.NewReader(dummy2) - - var buffer bytes.Buffer - wc := NewWriteCounter(&buffer) - - reader1.WriteTo(wc) - reader2.WriteTo(wc) - - if wc.Count != totalLength { - t.Errorf("Wrong count: %d vs. %d", wc.Count, totalLength) - } - - if buffer.String() != dummy1+dummy2 { - t.Error("Wrong message written") - } -} - func TestImageReference(t *testing.T) { tests := []struct { repo string @@ -152,3 +55,46 @@ func TestDigestReference(t *testing.T) { t.Errorf("Unexpected DigestReference=true for input %q", input) } } + +func TestReadDockerIgnore(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "dockerignore-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + diName := filepath.Join(tmpDir, ".dockerignore") + + di, err := ReadDockerIgnore(diName) + if err != nil { + t.Fatalf("Expected not to have error, got %s", err) + } + + if diLen := len(di); diLen != 0 { + t.Fatalf("Expected to have zero dockerignore entry, got %d", diLen) + } + + content := fmt.Sprintf("test1\n/test2\n/a/file/here\n\nlastfile") + err = ioutil.WriteFile(diName, []byte(content), 0777) + if err != nil { + t.Fatal(err) + } + + di, err = ReadDockerIgnore(diName) + if err != nil { + t.Fatal(err) + } + + if di[0] != "test1" { + t.Fatalf("First element is not test1") + } + if di[1] != "/test2" { + t.Fatalf("Second element is not /test2") + } + if di[2] != "/a/file/here" { + t.Fatalf("Third element is not /a/file/here") + } + if di[3] != "lastfile" { + t.Fatalf("Fourth element is not lastfile") + } +}