1package executor 2 3import ( 4 "fmt" 5 "os" 6 "os/exec" 7 "os/user" 8 "strconv" 9 "syscall" 10 11 "github.com/containernetworking/plugins/pkg/ns" 12 multierror "github.com/hashicorp/go-multierror" 13 "github.com/hashicorp/nomad/plugins/drivers" 14 "github.com/opencontainers/runc/libcontainer/cgroups" 15 cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs" 16 lconfigs "github.com/opencontainers/runc/libcontainer/configs" 17 "github.com/opencontainers/runc/libcontainer/specconv" 18) 19 20// setCmdUser takes a user id as a string and looks up the user, and sets the command 21// to execute as that user. 22func setCmdUser(cmd *exec.Cmd, userid string) error { 23 u, err := user.Lookup(userid) 24 if err != nil { 25 return fmt.Errorf("Failed to identify user %v: %v", userid, err) 26 } 27 28 // Get the groups the user is a part of 29 gidStrings, err := u.GroupIds() 30 if err != nil { 31 return fmt.Errorf("Unable to lookup user's group membership: %v", err) 32 } 33 34 gids := make([]uint32, len(gidStrings)) 35 for _, gidString := range gidStrings { 36 u, err := strconv.Atoi(gidString) 37 if err != nil { 38 return fmt.Errorf("Unable to convert user's group to int %s: %v", gidString, err) 39 } 40 41 gids = append(gids, uint32(u)) 42 } 43 44 // Convert the uid and gid 45 uid, err := strconv.ParseUint(u.Uid, 10, 32) 46 if err != nil { 47 return fmt.Errorf("Unable to convert userid to uint32: %s", err) 48 } 49 gid, err := strconv.ParseUint(u.Gid, 10, 32) 50 if err != nil { 51 return fmt.Errorf("Unable to convert groupid to uint32: %s", err) 52 } 53 54 // Set the command to run as that user and group. 55 if cmd.SysProcAttr == nil { 56 cmd.SysProcAttr = &syscall.SysProcAttr{} 57 } 58 if cmd.SysProcAttr.Credential == nil { 59 cmd.SysProcAttr.Credential = &syscall.Credential{} 60 } 61 cmd.SysProcAttr.Credential.Uid = uint32(uid) 62 cmd.SysProcAttr.Credential.Gid = uint32(gid) 63 cmd.SysProcAttr.Credential.Groups = gids 64 65 return nil 66} 67 68// configureResourceContainer configured the cgroups to be used to track pids 69// created by the executor 70func (e *UniversalExecutor) configureResourceContainer(pid int) error { 71 cfg := &lconfigs.Config{ 72 Cgroups: &lconfigs.Cgroup{ 73 Resources: &lconfigs.Resources{}, 74 }, 75 } 76 for _, device := range specconv.AllowedDevices { 77 cfg.Cgroups.Resources.Devices = append(cfg.Cgroups.Resources.Devices, &device.Rule) 78 } 79 80 err := configureBasicCgroups(cfg) 81 if err != nil { 82 // Log this error to help diagnose cases where nomad is run with too few 83 // permissions, but don't return an error. There is no separate check for 84 // cgroup creation permissions, so this may be the happy path. 85 e.logger.Warn("failed to create cgroup", 86 "docs", "https://www.nomadproject.io/docs/drivers/raw_exec.html#no_cgroups", 87 "error", err) 88 return nil 89 } 90 e.resConCtx.groups = cfg.Cgroups 91 return cgroups.EnterPid(cfg.Cgroups.Paths, pid) 92} 93 94func (e *UniversalExecutor) getAllPids() (map[int]*nomadPid, error) { 95 if e.resConCtx.isEmpty() { 96 return getAllPidsByScanning() 97 } else { 98 return e.resConCtx.getAllPidsByCgroup() 99 } 100} 101 102// DestroyCgroup kills all processes in the cgroup and removes the cgroup 103// configuration from the host. This function is idempotent. 104func DestroyCgroup(groups *lconfigs.Cgroup, executorPid int) error { 105 mErrs := new(multierror.Error) 106 if groups == nil { 107 return fmt.Errorf("Can't destroy: cgroup configuration empty") 108 } 109 110 // Move the executor into the global cgroup so that the task specific 111 // cgroup can be destroyed. 112 path, err := cgroups.GetInitCgroupPath("freezer") 113 if err != nil { 114 return err 115 } 116 117 if err := cgroups.EnterPid(map[string]string{"freezer": path}, executorPid); err != nil { 118 return err 119 } 120 121 // Freeze the Cgroup so that it can not continue to fork/exec. 122 groups.Resources.Freezer = lconfigs.Frozen 123 freezer := cgroupFs.FreezerGroup{} 124 if err := freezer.Set(groups.Paths[freezer.Name()], groups); err != nil { 125 return err 126 } 127 128 var procs []*os.Process 129 pids, err := cgroups.GetAllPids(groups.Paths[freezer.Name()]) 130 if err != nil { 131 multierror.Append(mErrs, fmt.Errorf("error getting pids: %v", err)) 132 133 // Unfreeze the cgroup. 134 groups.Resources.Freezer = lconfigs.Thawed 135 freezer := cgroupFs.FreezerGroup{} 136 if err := freezer.Set(groups.Paths[freezer.Name()], groups); err != nil { 137 multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err)) 138 return mErrs.ErrorOrNil() 139 } 140 } 141 142 // Kill the processes in the cgroup 143 for _, pid := range pids { 144 proc, err := os.FindProcess(pid) 145 if err != nil { 146 multierror.Append(mErrs, fmt.Errorf("error finding process %v: %v", pid, err)) 147 continue 148 } 149 150 procs = append(procs, proc) 151 if e := proc.Kill(); e != nil { 152 multierror.Append(mErrs, fmt.Errorf("error killing process %v: %v", pid, e)) 153 } 154 } 155 156 // Unfreeze the cgroug so we can wait. 157 groups.Resources.Freezer = lconfigs.Thawed 158 if err := freezer.Set(groups.Paths[freezer.Name()], groups); err != nil { 159 multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err)) 160 return mErrs.ErrorOrNil() 161 } 162 163 // Wait on the killed processes to ensure they are cleaned up. 164 for _, proc := range procs { 165 // Don't capture the error because we expect this to fail for 166 // processes we didn't fork. 167 proc.Wait() 168 } 169 170 // Remove the cgroup. 171 if err := cgroups.RemovePaths(groups.Paths); err != nil { 172 multierror.Append(mErrs, fmt.Errorf("failed to delete the cgroup directories: %v", err)) 173 } 174 return mErrs.ErrorOrNil() 175} 176 177// withNetworkIsolation calls the passed function the network namespace `spec` 178func withNetworkIsolation(f func() error, spec *drivers.NetworkIsolationSpec) error { 179 if spec != nil && spec.Path != "" { 180 // Get a handle to the target network namespace 181 netns, err := ns.GetNS(spec.Path) 182 if err != nil { 183 return err 184 } 185 186 // Start the container in the network namespace 187 return netns.Do(func(ns.NetNS) error { 188 return f() 189 }) 190 } 191 192 return f() 193} 194