1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00

Update libcontainer to 28cb5f9dfd6f3352c610a4f1502

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>
This commit is contained in:
Michael Crosby 2014-11-17 12:16:37 -08:00
parent ecaba0eb4a
commit 975fa5487c
16 changed files with 67 additions and 444 deletions

View file

@ -66,7 +66,7 @@ if [ "$1" = '--go' ]; then
mv tmp-tar src/code.google.com/p/go/src/pkg/archive/tar mv tmp-tar src/code.google.com/p/go/src/pkg/archive/tar
fi fi
clone git github.com/docker/libcontainer 4ae31b6ceb2c2557c9f05f42da61b0b808faa5a4 clone git github.com/docker/libcontainer 28cb5f9dfd6f3352c610a4f1502b5df4f69389ea
# see src/github.com/docker/libcontainer/update-vendor.sh which is the "source of truth" for libcontainer deps (just like this file) # see src/github.com/docker/libcontainer/update-vendor.sh which is the "source of truth" for libcontainer deps (just like this file)
rm -rf src/github.com/docker/libcontainer/vendor rm -rf src/github.com/docker/libcontainer/vendor
eval "$(grep '^clone ' src/github.com/docker/libcontainer/update-vendor.sh | grep -v 'github.com/codegangsta/cli')" eval "$(grep '^clone ' src/github.com/docker/libcontainer/update-vendor.sh | grep -v 'github.com/codegangsta/cli')"

View file

@ -1,7 +1,7 @@
FROM crosbymichael/golang FROM crosbymichael/golang
RUN apt-get update && apt-get install -y gcc make RUN apt-get update && apt-get install -y gcc make
RUN go get code.google.com/p/go.tools/cmd/cover RUN go get golang.org/x/tools/cmd/cover
ENV GOPATH $GOPATH:/go/src/github.com/docker/libcontainer/vendor ENV GOPATH $GOPATH:/go/src/github.com/docker/libcontainer/vendor
RUN go get github.com/docker/docker/pkg/term RUN go get github.com/docker/docker/pkg/term
@ -10,7 +10,7 @@ RUN go get github.com/docker/docker/pkg/term
RUN mkdir /busybox && \ RUN mkdir /busybox && \
curl -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.02/rootfs.tar' | tar -xC /busybox curl -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.02/rootfs.tar' | tar -xC /busybox
RUN curl -sSL https://raw.githubusercontent.com/docker/docker/master/hack/dind -o /dind && \ RUN curl -sSL https://raw.githubusercontent.com/docker/docker/master/project/dind -o /dind && \
chmod +x /dind chmod +x /dind
COPY . /go/src/github.com/docker/libcontainer COPY . /go/src/github.com/docker/libcontainer

View file

@ -5,30 +5,17 @@ package libcontainer
import ( import (
"github.com/docker/libcontainer/cgroups/fs" "github.com/docker/libcontainer/cgroups/fs"
"github.com/docker/libcontainer/cgroups/systemd"
"github.com/docker/libcontainer/network" "github.com/docker/libcontainer/network"
) )
// TODO(vmarmol): Complete Stats() in final libcontainer API and move users to that. // TODO(vmarmol): Complete Stats() in final libcontainer API and move users to that.
// DEPRECATED: The below portions are only to be used during the transition to the official API. // DEPRECATED: The below portions are only to be used during the transition to the official API.
// Returns all available stats for the given container. // Returns all available stats for the given container.
func GetStats(container *Config, state *State) (*ContainerStats, error) { func GetStats(container *Config, state *State) (stats *ContainerStats, err error) {
var (
err error
stats = &ContainerStats{} stats = &ContainerStats{}
) if stats.CgroupStats, err = fs.GetStats(state.CgroupPaths); err != nil {
if systemd.UseSystemd() {
stats.CgroupStats, err = systemd.GetStats(container.Cgroups)
} else {
stats.CgroupStats, err = fs.GetStats(container.Cgroups)
}
if err != nil {
return stats, err return stats, err
} }
stats.NetworkStats, err = network.GetStats(&state.NetworkState) stats.NetworkStats, err = network.GetStats(&state.NetworkState)
return stats, err return stats, err
} }

View file

@ -53,8 +53,3 @@ type Cgroup struct {
Freezer FreezerState `json:"freezer,omitempty"` // set the freeze value for the process Freezer FreezerState `json:"freezer,omitempty"` // set the freeze value for the process
Slice string `json:"slice,omitempty"` // Parent slice to use for systemd Slice string `json:"slice,omitempty"` // Parent slice to use for systemd
} }
type ActiveCgroup interface {
Cleanup() error
Paths() (map[string]string, error)
}

View file

@ -1,264 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"log"
"os"
"syscall"
"time"
"github.com/codegangsta/cli"
"github.com/docker/libcontainer/cgroups"
"github.com/docker/libcontainer/cgroups/fs"
"github.com/docker/libcontainer/cgroups/systemd"
)
var createCommand = cli.Command{
Name: "create",
Usage: "Create a cgroup container using the supplied configuration and initial process.",
Flags: []cli.Flag{
cli.StringFlag{Name: "config, c", Value: "cgroup.json", Usage: "path to container configuration (cgroups.Cgroup object)"},
cli.IntFlag{Name: "pid, p", Value: 0, Usage: "pid of the initial process in the container"},
},
Action: createAction,
}
var destroyCommand = cli.Command{
Name: "destroy",
Usage: "Destroy an existing cgroup container.",
Flags: []cli.Flag{
cli.StringFlag{Name: "name, n", Value: "", Usage: "container name"},
cli.StringFlag{Name: "parent, p", Value: "", Usage: "container parent"},
},
Action: destroyAction,
}
var statsCommand = cli.Command{
Name: "stats",
Usage: "Get stats for cgroup",
Flags: []cli.Flag{
cli.StringFlag{Name: "name, n", Value: "", Usage: "container name"},
cli.StringFlag{Name: "parent, p", Value: "", Usage: "container parent"},
},
Action: statsAction,
}
var pauseCommand = cli.Command{
Name: "pause",
Usage: "Pause cgroup",
Flags: []cli.Flag{
cli.StringFlag{Name: "name, n", Value: "", Usage: "container name"},
cli.StringFlag{Name: "parent, p", Value: "", Usage: "container parent"},
},
Action: pauseAction,
}
var resumeCommand = cli.Command{
Name: "resume",
Usage: "Resume a paused cgroup",
Flags: []cli.Flag{
cli.StringFlag{Name: "name, n", Value: "", Usage: "container name"},
cli.StringFlag{Name: "parent, p", Value: "", Usage: "container parent"},
},
Action: resumeAction,
}
var psCommand = cli.Command{
Name: "ps",
Usage: "Get list of pids for a cgroup",
Flags: []cli.Flag{
cli.StringFlag{Name: "name, n", Value: "", Usage: "container name"},
cli.StringFlag{Name: "parent, p", Value: "", Usage: "container parent"},
},
Action: psAction,
}
func getConfigFromFile(c *cli.Context) (*cgroups.Cgroup, error) {
f, err := os.Open(c.String("config"))
if err != nil {
return nil, err
}
defer f.Close()
var config *cgroups.Cgroup
if err := json.NewDecoder(f).Decode(&config); err != nil {
log.Fatal(err)
}
return config, nil
}
func openLog(name string) error {
f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0755)
if err != nil {
return err
}
log.SetOutput(f)
return nil
}
func getConfig(context *cli.Context) (*cgroups.Cgroup, error) {
name := context.String("name")
if name == "" {
log.Fatal(fmt.Errorf("Missing container name"))
}
parent := context.String("parent")
return &cgroups.Cgroup{
Name: name,
Parent: parent,
}, nil
}
func killAll(config *cgroups.Cgroup) {
// We could use freezer here to prevent process spawning while we are trying
// to kill everything. But going with more portable solution of retrying for
// now.
pids := getPids(config)
retry := 10
for len(pids) != 0 || retry > 0 {
killPids(pids)
time.Sleep(100 * time.Millisecond)
retry--
pids = getPids(config)
}
if len(pids) != 0 {
log.Fatal(fmt.Errorf("Could not kill existing processes in the container."))
}
}
func getPids(config *cgroups.Cgroup) []int {
pids, err := fs.GetPids(config)
if err != nil {
log.Fatal(err)
}
return pids
}
func killPids(pids []int) {
for _, pid := range pids {
// pids might go away on their own. Ignore errors.
syscall.Kill(pid, syscall.SIGKILL)
}
}
func setFreezerState(context *cli.Context, state cgroups.FreezerState) {
config, err := getConfig(context)
if err != nil {
log.Fatal(err)
}
if systemd.UseSystemd() {
err = systemd.Freeze(config, state)
} else {
err = fs.Freeze(config, state)
}
if err != nil {
log.Fatal(err)
}
}
func createAction(context *cli.Context) {
config, err := getConfigFromFile(context)
if err != nil {
log.Fatal(err)
}
pid := context.Int("pid")
if pid <= 0 {
log.Fatal(fmt.Errorf("Invalid pid : %d", pid))
}
if systemd.UseSystemd() {
_, err := systemd.Apply(config, pid)
if err != nil {
log.Fatal(err)
}
} else {
_, err := fs.Apply(config, pid)
if err != nil {
log.Fatal(err)
}
}
}
func destroyAction(context *cli.Context) {
config, err := getConfig(context)
if err != nil {
log.Fatal(err)
}
killAll(config)
// Systemd will clean up cgroup state for empty container.
if !systemd.UseSystemd() {
err := fs.Cleanup(config)
if err != nil {
log.Fatal(err)
}
}
}
func statsAction(context *cli.Context) {
config, err := getConfig(context)
if err != nil {
log.Fatal(err)
}
stats, err := fs.GetStats(config)
if err != nil {
log.Fatal(err)
}
out, err := json.MarshalIndent(stats, "", "\t")
if err != nil {
log.Fatal(err)
}
fmt.Printf("Usage stats for '%s':\n %v\n", config.Name, string(out))
}
func pauseAction(context *cli.Context) {
setFreezerState(context, cgroups.Frozen)
}
func resumeAction(context *cli.Context) {
setFreezerState(context, cgroups.Thawed)
}
func psAction(context *cli.Context) {
config, err := getConfig(context)
if err != nil {
log.Fatal(err)
}
pids, err := fs.GetPids(config)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Pids in '%s':\n", config.Name)
fmt.Println(pids)
}
func main() {
logPath := os.Getenv("log")
if logPath != "" {
if err := openLog(logPath); err != nil {
log.Fatal(err)
}
}
app := cli.NewApp()
app.Name = "cgutil"
app.Usage = "Test utility for libcontainer cgroups package"
app.Version = "0.1"
app.Commands = []cli.Command{
createCommand,
destroyCommand,
statsCommand,
pauseCommand,
resumeCommand,
psCommand,
}
if err := app.Run(os.Args); err != nil {
log.Fatal(err)
}
}

View file

@ -1,10 +0,0 @@
{
"name": "luke",
"parent": "darth",
"allow_all_devices": true,
"memory": 1073741824,
"memory_swap": -1,
"cpu_shares": 2048,
"cpu_quota": 500000,
"cpu_period": 250000
}

View file

@ -57,20 +57,35 @@ type data struct {
pid int pid int
} }
func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) { func Apply(c *cgroups.Cgroup, pid int) (map[string]string, error) {
d, err := getCgroupData(c, pid) d, err := getCgroupData(c, pid)
if err != nil { if err != nil {
return nil, err return nil, err
} }
for _, sys := range subsystems { paths := make(map[string]string)
defer func() {
if err != nil {
cgroups.RemovePaths(paths)
}
}()
for name, sys := range subsystems {
if err := sys.Set(d); err != nil { if err := sys.Set(d); err != nil {
d.Cleanup()
return nil, err return nil, err
} }
// FIXME: Apply should, ideally, be reentrant or be broken up into a separate
// create and join phase so that the cgroup hierarchy for a container can be
// created then join consists of writing the process pids to cgroup.procs
p, err := d.path(name)
if err != nil {
if cgroups.IsNotFound(err) {
continue
} }
return nil, err
return d, nil }
paths[name] = p
}
return paths, nil
} }
// Symmetrical public function to update device based cgroups. Also available // Symmetrical public function to update device based cgroups. Also available
@ -86,33 +101,13 @@ func ApplyDevices(c *cgroups.Cgroup, pid int) error {
return devices.Set(d) return devices.Set(d)
} }
func Cleanup(c *cgroups.Cgroup) error { func GetStats(systemPaths map[string]string) (*cgroups.Stats, error) {
d, err := getCgroupData(c, 0)
if err != nil {
return fmt.Errorf("Could not get Cgroup data %s", err)
}
return d.Cleanup()
}
func GetStats(c *cgroups.Cgroup) (*cgroups.Stats, error) {
stats := cgroups.NewStats() stats := cgroups.NewStats()
for name, path := range systemPaths {
d, err := getCgroupData(c, 0) sys, ok := subsystems[name]
if err != nil { if !ok {
return nil, fmt.Errorf("getting CgroupData %s", err)
}
for sysname, sys := range subsystems {
path, err := d.path(sysname)
if err != nil {
// Don't fail if a cgroup hierarchy was not found, just skip this subsystem
if cgroups.IsNotFound(err) {
continue continue
} }
return nil, err
}
if err := sys.GetStats(path, stats); err != nil { if err := sys.GetStats(path, stats); err != nil {
return nil, err return nil, err
} }
@ -176,26 +171,6 @@ func (raw *data) parent(subsystem string) (string, error) {
return filepath.Join(raw.root, subsystem, initPath), nil return filepath.Join(raw.root, subsystem, initPath), nil
} }
func (raw *data) Paths() (map[string]string, error) {
paths := make(map[string]string)
for sysname := range subsystems {
path, err := raw.path(sysname)
if err != nil {
// Don't fail if a cgroup hierarchy was not found, just skip this subsystem
if cgroups.IsNotFound(err) {
continue
}
return nil, err
}
paths[sysname] = path
}
return paths, nil
}
func (raw *data) path(subsystem string) (string, error) { func (raw *data) path(subsystem string) (string, error) {
// If the cgroup name/path is absolute do not look relative to the cgroup of the init process. // If the cgroup name/path is absolute do not look relative to the cgroup of the init process.
if filepath.IsAbs(raw.cgroup) { if filepath.IsAbs(raw.cgroup) {
@ -234,13 +209,6 @@ func (raw *data) join(subsystem string) (string, error) {
return path, nil return path, nil
} }
func (raw *data) Cleanup() error {
for _, sys := range subsystems {
sys.Remove(raw)
}
return nil
}
func writeFile(dir, file, data string) error { func writeFile(dir, file, data string) error {
return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700)
} }

View file

@ -27,7 +27,7 @@ type CpuUsage struct {
type CpuStats struct { type CpuStats struct {
CpuUsage CpuUsage `json:"cpu_usage,omitempty"` CpuUsage CpuUsage `json:"cpu_usage,omitempty"`
ThrottlingData ThrottlingData `json:"throlling_data,omitempty"` ThrottlingData ThrottlingData `json:"throttling_data,omitempty"`
} }
type MemoryStats struct { type MemoryStats struct {

View file

@ -12,7 +12,7 @@ func UseSystemd() bool {
return false return false
} }
func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) { func Apply(c *cgroups.Cgroup, pid int) (map[string]string, error) {
return nil, fmt.Errorf("Systemd not supported") return nil, fmt.Errorf("Systemd not supported")
} }
@ -27,7 +27,3 @@ func ApplyDevices(c *cgroups.Cgroup, pid int) error {
func Freeze(c *cgroups.Cgroup, state cgroups.FreezerState) error { func Freeze(c *cgroups.Cgroup, state cgroups.FreezerState) error {
return fmt.Errorf("Systemd not supported") return fmt.Errorf("Systemd not supported")
} }
func GetStats(c *cgroups.Cgroup) (*cgroups.Stats, error) {
return nil, fmt.Errorf("Systemd not supported")
}

View file

@ -31,16 +31,6 @@ var (
connLock sync.Mutex connLock sync.Mutex
theConn *systemd.Conn theConn *systemd.Conn
hasStartTransientUnit bool hasStartTransientUnit bool
subsystems = map[string]subsystem{
"devices": &fs.DevicesGroup{},
"memory": &fs.MemoryGroup{},
"cpu": &fs.CpuGroup{},
"cpuset": &fs.CpusetGroup{},
"cpuacct": &fs.CpuacctGroup{},
"blkio": &fs.BlkioGroup{},
"perf_event": &fs.PerfEventGroup{},
"freezer": &fs.FreezerGroup{},
}
) )
func newProp(name string, units interface{}) systemd.Property { func newProp(name string, units interface{}) systemd.Property {
@ -91,7 +81,7 @@ func getIfaceForUnit(unitName string) string {
return "Unit" return "Unit"
} }
func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) { func Apply(c *cgroups.Cgroup, pid int) (map[string]string, error) {
var ( var (
unitName = getUnitName(c) unitName = getUnitName(c)
slice = "system.slice" slice = "system.slice"
@ -159,45 +149,32 @@ func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) {
} }
} }
return res, nil
}
func writeFile(dir, file, data string) error {
return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700)
}
func (c *systemdCgroup) Paths() (map[string]string, error) {
paths := make(map[string]string) paths := make(map[string]string)
for _, sysname := range []string{
for sysname := range subsystems { "devices",
subsystemPath, err := getSubsystemPath(c.cgroup, sysname) "memory",
"cpu",
"cpuset",
"cpuacct",
"blkio",
"perf_event",
"freezer",
} {
subsystemPath, err := getSubsystemPath(res.cgroup, sysname)
if err != nil { if err != nil {
// Don't fail if a cgroup hierarchy was not found, just skip this subsystem // Don't fail if a cgroup hierarchy was not found, just skip this subsystem
if cgroups.IsNotFound(err) { if cgroups.IsNotFound(err) {
continue continue
} }
return nil, err return nil, err
} }
paths[sysname] = subsystemPath paths[sysname] = subsystemPath
} }
return paths, nil return paths, nil
} }
func (c *systemdCgroup) Cleanup() error { func writeFile(dir, file, data string) error {
// systemd cleans up, we don't need to do much return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700)
paths, err := c.Paths()
if err != nil {
return err
}
for _, path := range paths {
os.RemoveAll(path)
}
return nil
} }
func joinFreezer(c *cgroups.Cgroup, pid int) error { func joinFreezer(c *cgroups.Cgroup, pid int) error {
@ -267,35 +244,6 @@ func getUnitName(c *cgroups.Cgroup) string {
return fmt.Sprintf("%s-%s.scope", c.Parent, c.Name) return fmt.Sprintf("%s-%s.scope", c.Parent, c.Name)
} }
/*
* This would be nicer to get from the systemd API when accounting
* is enabled, but sadly there is no way to do that yet.
* The lack of this functionality in the API & the approach taken
* is guided by
* http://www.freedesktop.org/wiki/Software/systemd/ControlGroupInterface/#readingaccountinginformation.
*/
func GetStats(c *cgroups.Cgroup) (*cgroups.Stats, error) {
stats := cgroups.NewStats()
for sysname, sys := range subsystems {
subsystemPath, err := getSubsystemPath(c, sysname)
if err != nil {
// Don't fail if a cgroup hierarchy was not found, just skip this subsystem
if cgroups.IsNotFound(err) {
continue
}
return nil, err
}
if err := sys.GetStats(subsystemPath, stats); err != nil {
return nil, err
}
}
return stats, nil
}
// Atm we can't use the systemd device support because of two missing things: // Atm we can't use the systemd device support because of two missing things:
// * Support for wildcards to allow mknod on any device // * Support for wildcards to allow mknod on any device
// * Support for wildcards to allow /dev/pts support // * Support for wildcards to allow /dev/pts support

View file

@ -189,6 +189,17 @@ func EnterPid(cgroupPaths map[string]string, pid int) error {
} }
} }
} }
return nil return nil
} }
// RemovePaths iterates over the provided paths removing them.
// If an error is encountered the removal proceeds and the first error is
// returned to ensure a partial removal is not possible.
func RemovePaths(paths map[string]string) (err error) {
for _, path := range paths {
if rerr := os.RemoveAll(path); err == nil {
err = rerr
}
}
return err
}

View file

@ -60,16 +60,11 @@ func Exec(container *libcontainer.Config, stdin io.Reader, stdout, stderr io.Wri
// Do this before syncing with child so that no children // Do this before syncing with child so that no children
// can escape the cgroup // can escape the cgroup
cgroupRef, err := SetupCgroups(container, command.Process.Pid) cgroupPaths, err := SetupCgroups(container, command.Process.Pid)
if err != nil {
return terminate(err)
}
defer cgroupRef.Cleanup()
cgroupPaths, err := cgroupRef.Paths()
if err != nil { if err != nil {
return terminate(err) return terminate(err)
} }
defer cgroups.RemovePaths(cgroupPaths)
var networkState network.NetworkState var networkState network.NetworkState
if err := InitializeNetworking(container, command.Process.Pid, &networkState); err != nil { if err := InitializeNetworking(container, command.Process.Pid, &networkState); err != nil {
@ -153,18 +148,15 @@ func DefaultCreateCommand(container *libcontainer.Config, console, dataPath, ini
// SetupCgroups applies the cgroup restrictions to the process running in the container based // SetupCgroups applies the cgroup restrictions to the process running in the container based
// on the container's configuration // on the container's configuration
func SetupCgroups(container *libcontainer.Config, nspid int) (cgroups.ActiveCgroup, error) { func SetupCgroups(container *libcontainer.Config, nspid int) (map[string]string, error) {
if container.Cgroups != nil { if container.Cgroups != nil {
c := container.Cgroups c := container.Cgroups
if systemd.UseSystemd() { if systemd.UseSystemd() {
return systemd.Apply(c, nspid) return systemd.Apply(c, nspid)
} }
return fs.Apply(c, nspid) return fs.Apply(c, nspid)
} }
return map[string]string{}, nil
return nil, nil
} }
// InitializeNetworking creates the container's network stack outside of the namespace and moves // InitializeNetworking creates the container's network stack outside of the namespace and moves

View file

@ -111,7 +111,7 @@ func FinalizeSetns(container *libcontainer.Config, args []string) error {
} }
} }
if err := system.Execv(args[0], args[0:], container.Env); err != nil { if err := system.Execv(args[0], args[0:], os.Environ()); err != nil {
return err return err
} }

View file

@ -100,7 +100,7 @@ func Init(container *libcontainer.Config, uncleanRootfs, consolePath string, pip
if container.Hostname != "" { if container.Hostname != "" {
if err := syscall.Sethostname([]byte(container.Hostname)); err != nil { if err := syscall.Sethostname([]byte(container.Hostname)); err != nil {
return fmt.Errorf("sethostname %s", err) return fmt.Errorf("unable to sethostname %q: %s", container.Hostname, err)
} }
} }

View file

@ -1004,7 +1004,7 @@ func AddRoute(destination, source, gateway, device string) error {
if source != "" { if source != "" {
srcIP := net.ParseIP(source) srcIP := net.ParseIP(source)
if err != nil { if srcIP == nil {
return fmt.Errorf("source IP %s couldn't be parsed", source) return fmt.Errorf("source IP %s couldn't be parsed", source)
} }
srcFamily := getIpFamily(srcIP) srcFamily := getIpFamily(srcIP)

View file

@ -7,7 +7,7 @@ import (
// Setuid sets the uid of the calling thread to the specified uid. // Setuid sets the uid of the calling thread to the specified uid.
func Setuid(uid int) (err error) { func Setuid(uid int) (err error) {
_, _, e1 := syscall.RawSyscall(syscall.SYS_SETUID, uintptr(uid), 0, 0) _, _, e1 := syscall.RawSyscall(syscall.SYS_SETUID32, uintptr(uid), 0, 0)
if e1 != 0 { if e1 != 0 {
err = e1 err = e1
} }