1// Copyright 2011 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// +build linux 6 7package syscall 8 9import ( 10 "runtime" 11 "unsafe" 12) 13 14// SysProcIDMap holds Container ID to Host ID mappings used for User Namespaces in Linux. 15// See user_namespaces(7). 16type SysProcIDMap struct { 17 ContainerID int // Container ID. 18 HostID int // Host ID. 19 Size int // Size. 20} 21 22type SysProcAttr struct { 23 Chroot string // Chroot. 24 Credential *Credential // Credential. 25 // Ptrace tells the child to call ptrace(PTRACE_TRACEME). 26 // Call runtime.LockOSThread before starting a process with this set, 27 // and don't call UnlockOSThread until done with PtraceSyscall calls. 28 Ptrace bool 29 Setsid bool // Create session. 30 Setpgid bool // Set process group ID to Pgid, or, if Pgid == 0, to new pid. 31 Setctty bool // Set controlling terminal to fd Ctty (only meaningful if Setsid is set) 32 Noctty bool // Detach fd 0 from controlling terminal 33 Ctty int // Controlling TTY fd 34 Foreground bool // Place child's process group in foreground. (Implies Setpgid. Uses Ctty as fd of controlling TTY) 35 Pgid int // Child's process group ID if Setpgid. 36 Pdeathsig Signal // Signal that the process will get when its parent dies (Linux only) 37 Cloneflags uintptr // Flags for clone calls (Linux only) 38 Unshareflags uintptr // Flags for unshare calls (Linux only) 39 UidMappings []SysProcIDMap // User ID mappings for user namespaces. 40 GidMappings []SysProcIDMap // Group ID mappings for user namespaces. 41 // GidMappingsEnableSetgroups enabling setgroups syscall. 42 // If false, then setgroups syscall will be disabled for the child process. 43 // This parameter is no-op if GidMappings == nil. Otherwise for unprivileged 44 // users this should be set to false for mappings work. 45 GidMappingsEnableSetgroups bool 46 AmbientCaps []uintptr // Ambient capabilities (Linux only) 47} 48 49var ( 50 none = [...]byte{'n', 'o', 'n', 'e', 0} 51 slash = [...]byte{'/', 0} 52) 53 54// Implemented in runtime package. 55func runtime_BeforeFork() 56func runtime_AfterFork() 57func runtime_AfterForkInChild() 58 59// Fork, dup fd onto 0..len(fd), and exec(argv0, argvv, envv) in child. 60// If a dup or exec fails, write the errno error to pipe. 61// (Pipe is close-on-exec so if exec succeeds, it will be closed.) 62// In the child, this function must not acquire any locks, because 63// they might have been locked at the time of the fork. This means 64// no rescheduling, no malloc calls, and no new stack segments. 65// For the same reason compiler does not race instrument it. 66// The calls to RawSyscall are okay because they are assembly 67// functions that do not grow the stack. 68//go:norace 69func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr *ProcAttr, sys *SysProcAttr, pipe int) (pid int, err Errno) { 70 // Set up and fork. This returns immediately in the parent or 71 // if there's an error. 72 r1, err1, p, locked := forkAndExecInChild1(argv0, argv, envv, chroot, dir, attr, sys, pipe) 73 if locked { 74 runtime_AfterFork() 75 } 76 if err1 != 0 { 77 return 0, err1 78 } 79 80 // parent; return PID 81 pid = int(r1) 82 83 if sys.UidMappings != nil || sys.GidMappings != nil { 84 Close(p[0]) 85 var err2 Errno 86 // uid/gid mappings will be written after fork and unshare(2) for user 87 // namespaces. 88 if sys.Unshareflags&CLONE_NEWUSER == 0 { 89 if err := writeUidGidMappings(pid, sys); err != nil { 90 err2 = err.(Errno) 91 } 92 } 93 RawSyscall(SYS_WRITE, uintptr(p[1]), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2)) 94 Close(p[1]) 95 } 96 97 return pid, 0 98} 99 100const _LINUX_CAPABILITY_VERSION_3 = 0x20080522 101 102type capHeader struct { 103 version uint32 104 pid int32 105} 106 107type capData struct { 108 effective uint32 109 permitted uint32 110 inheritable uint32 111} 112type caps struct { 113 hdr capHeader 114 data [2]capData 115} 116 117// See CAP_TO_INDEX in linux/capability.h: 118func capToIndex(cap uintptr) uintptr { return cap >> 5 } 119 120// See CAP_TO_MASK in linux/capability.h: 121func capToMask(cap uintptr) uint32 { return 1 << uint(cap&31) } 122 123// forkAndExecInChild1 implements the body of forkAndExecInChild up to 124// the parent's post-fork path. This is a separate function so we can 125// separate the child's and parent's stack frames if we're using 126// vfork. 127// 128// This is go:noinline because the point is to keep the stack frames 129// of this and forkAndExecInChild separate. 130// 131//go:noinline 132//go:norace 133func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr *ProcAttr, sys *SysProcAttr, pipe int) (r1 uintptr, err1 Errno, p [2]int, locked bool) { 134 // Defined in linux/prctl.h starting with Linux 4.3. 135 const ( 136 PR_CAP_AMBIENT = 0x2f 137 PR_CAP_AMBIENT_RAISE = 0x2 138 ) 139 140 // vfork requires that the child not touch any of the parent's 141 // active stack frames. Hence, the child does all post-fork 142 // processing in this stack frame and never returns, while the 143 // parent returns immediately from this frame and does all 144 // post-fork processing in the outer frame. 145 // Declare all variables at top in case any 146 // declarations require heap allocation (e.g., err1). 147 var ( 148 err2 Errno 149 nextfd int 150 i int 151 caps caps 152 fd1 uintptr 153 puid, psetgroups, pgid []byte 154 uidmap, setgroups, gidmap []byte 155 ) 156 157 if sys.UidMappings != nil { 158 puid = []byte("/proc/self/uid_map\000") 159 uidmap = formatIDMappings(sys.UidMappings) 160 } 161 162 if sys.GidMappings != nil { 163 psetgroups = []byte("/proc/self/setgroups\000") 164 pgid = []byte("/proc/self/gid_map\000") 165 166 if sys.GidMappingsEnableSetgroups { 167 setgroups = []byte("allow\000") 168 } else { 169 setgroups = []byte("deny\000") 170 } 171 gidmap = formatIDMappings(sys.GidMappings) 172 } 173 174 // Record parent PID so child can test if it has died. 175 ppid, _ := rawSyscallNoError(SYS_GETPID, 0, 0, 0) 176 177 // Guard against side effects of shuffling fds below. 178 // Make sure that nextfd is beyond any currently open files so 179 // that we can't run the risk of overwriting any of them. 180 fd := make([]int, len(attr.Files)) 181 nextfd = len(attr.Files) 182 for i, ufd := range attr.Files { 183 if nextfd < int(ufd) { 184 nextfd = int(ufd) 185 } 186 fd[i] = int(ufd) 187 } 188 nextfd++ 189 190 // Allocate another pipe for parent to child communication for 191 // synchronizing writing of User ID/Group ID mappings. 192 if sys.UidMappings != nil || sys.GidMappings != nil { 193 if err := forkExecPipe(p[:]); err != nil { 194 err1 = err.(Errno) 195 return 196 } 197 } 198 199 hasRawVforkSyscall := runtime.GOARCH == "amd64" || runtime.GOARCH == "ppc64" || runtime.GOARCH == "s390x" || runtime.GOARCH == "arm64" 200 201 // About to call fork. 202 // No more allocation or calls of non-assembly functions. 203 runtime_BeforeFork() 204 locked = true 205 switch { 206 case hasRawVforkSyscall && (sys.Cloneflags&CLONE_NEWUSER == 0 && sys.Unshareflags&CLONE_NEWUSER == 0): 207 r1, err1 = rawVforkSyscall(SYS_CLONE, uintptr(SIGCHLD|CLONE_VFORK|CLONE_VM)|sys.Cloneflags) 208 case runtime.GOARCH == "s390x": 209 r1, _, err1 = RawSyscall6(SYS_CLONE, 0, uintptr(SIGCHLD)|sys.Cloneflags, 0, 0, 0, 0) 210 default: 211 r1, _, err1 = RawSyscall6(SYS_CLONE, uintptr(SIGCHLD)|sys.Cloneflags, 0, 0, 0, 0, 0) 212 } 213 if err1 != 0 || r1 != 0 { 214 // If we're in the parent, we must return immediately 215 // so we're not in the same stack frame as the child. 216 // This can at most use the return PC, which the child 217 // will not modify, and the results of 218 // rawVforkSyscall, which must have been written after 219 // the child was replaced. 220 return 221 } 222 223 // Fork succeeded, now in child. 224 225 runtime_AfterForkInChild() 226 227 // Enable the "keep capabilities" flag to set ambient capabilities later. 228 if len(sys.AmbientCaps) > 0 { 229 _, _, err1 = RawSyscall6(SYS_PRCTL, PR_SET_KEEPCAPS, 1, 0, 0, 0, 0) 230 if err1 != 0 { 231 goto childerror 232 } 233 } 234 235 // Wait for User ID/Group ID mappings to be written. 236 if sys.UidMappings != nil || sys.GidMappings != nil { 237 if _, _, err1 = RawSyscall(SYS_CLOSE, uintptr(p[1]), 0, 0); err1 != 0 { 238 goto childerror 239 } 240 r1, _, err1 = RawSyscall(SYS_READ, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2)) 241 if err1 != 0 { 242 goto childerror 243 } 244 if r1 != unsafe.Sizeof(err2) { 245 err1 = EINVAL 246 goto childerror 247 } 248 if err2 != 0 { 249 err1 = err2 250 goto childerror 251 } 252 } 253 254 // Session ID 255 if sys.Setsid { 256 _, _, err1 = RawSyscall(SYS_SETSID, 0, 0, 0) 257 if err1 != 0 { 258 goto childerror 259 } 260 } 261 262 // Set process group 263 if sys.Setpgid || sys.Foreground { 264 // Place child in process group. 265 _, _, err1 = RawSyscall(SYS_SETPGID, 0, uintptr(sys.Pgid), 0) 266 if err1 != 0 { 267 goto childerror 268 } 269 } 270 271 if sys.Foreground { 272 pgrp := int32(sys.Pgid) 273 if pgrp == 0 { 274 r1, _ = rawSyscallNoError(SYS_GETPID, 0, 0, 0) 275 276 pgrp = int32(r1) 277 } 278 279 // Place process group in foreground. 280 _, _, err1 = RawSyscall(SYS_IOCTL, uintptr(sys.Ctty), uintptr(TIOCSPGRP), uintptr(unsafe.Pointer(&pgrp))) 281 if err1 != 0 { 282 goto childerror 283 } 284 } 285 286 // Unshare 287 if sys.Unshareflags != 0 { 288 _, _, err1 = RawSyscall(SYS_UNSHARE, sys.Unshareflags, 0, 0) 289 if err1 != 0 { 290 goto childerror 291 } 292 293 if sys.Unshareflags&CLONE_NEWUSER != 0 && sys.GidMappings != nil { 294 dirfd := int(_AT_FDCWD) 295 if fd1, _, err1 = RawSyscall6(SYS_OPENAT, uintptr(dirfd), uintptr(unsafe.Pointer(&psetgroups[0])), uintptr(O_WRONLY), 0, 0, 0); err1 != 0 { 296 goto childerror 297 } 298 r1, _, err1 = RawSyscall(SYS_WRITE, uintptr(fd1), uintptr(unsafe.Pointer(&setgroups[0])), uintptr(len(setgroups))) 299 if err1 != 0 { 300 goto childerror 301 } 302 if _, _, err1 = RawSyscall(SYS_CLOSE, uintptr(fd1), 0, 0); err1 != 0 { 303 goto childerror 304 } 305 306 if fd1, _, err1 = RawSyscall6(SYS_OPENAT, uintptr(dirfd), uintptr(unsafe.Pointer(&pgid[0])), uintptr(O_WRONLY), 0, 0, 0); err1 != 0 { 307 goto childerror 308 } 309 r1, _, err1 = RawSyscall(SYS_WRITE, uintptr(fd1), uintptr(unsafe.Pointer(&gidmap[0])), uintptr(len(gidmap))) 310 if err1 != 0 { 311 goto childerror 312 } 313 if _, _, err1 = RawSyscall(SYS_CLOSE, uintptr(fd1), 0, 0); err1 != 0 { 314 goto childerror 315 } 316 } 317 318 if sys.Unshareflags&CLONE_NEWUSER != 0 && sys.UidMappings != nil { 319 dirfd := int(_AT_FDCWD) 320 if fd1, _, err1 = RawSyscall6(SYS_OPENAT, uintptr(dirfd), uintptr(unsafe.Pointer(&puid[0])), uintptr(O_WRONLY), 0, 0, 0); err1 != 0 { 321 goto childerror 322 } 323 r1, _, err1 = RawSyscall(SYS_WRITE, uintptr(fd1), uintptr(unsafe.Pointer(&uidmap[0])), uintptr(len(uidmap))) 324 if err1 != 0 { 325 goto childerror 326 } 327 if _, _, err1 = RawSyscall(SYS_CLOSE, uintptr(fd1), 0, 0); err1 != 0 { 328 goto childerror 329 } 330 } 331 332 // The unshare system call in Linux doesn't unshare mount points 333 // mounted with --shared. Systemd mounts / with --shared. For a 334 // long discussion of the pros and cons of this see debian bug 739593. 335 // The Go model of unsharing is more like Plan 9, where you ask 336 // to unshare and the namespaces are unconditionally unshared. 337 // To make this model work we must further mark / as MS_PRIVATE. 338 // This is what the standard unshare command does. 339 if sys.Unshareflags&CLONE_NEWNS == CLONE_NEWNS { 340 _, _, err1 = RawSyscall6(SYS_MOUNT, uintptr(unsafe.Pointer(&none[0])), uintptr(unsafe.Pointer(&slash[0])), 0, MS_REC|MS_PRIVATE, 0, 0) 341 if err1 != 0 { 342 goto childerror 343 } 344 } 345 } 346 347 // Chroot 348 if chroot != nil { 349 _, _, err1 = RawSyscall(SYS_CHROOT, uintptr(unsafe.Pointer(chroot)), 0, 0) 350 if err1 != 0 { 351 goto childerror 352 } 353 } 354 355 // User and groups 356 if cred := sys.Credential; cred != nil { 357 ngroups := uintptr(len(cred.Groups)) 358 groups := uintptr(0) 359 if ngroups > 0 { 360 groups = uintptr(unsafe.Pointer(&cred.Groups[0])) 361 } 362 if !(sys.GidMappings != nil && !sys.GidMappingsEnableSetgroups && ngroups == 0) && !cred.NoSetGroups { 363 _, _, err1 = RawSyscall(_SYS_setgroups, ngroups, groups, 0) 364 if err1 != 0 { 365 goto childerror 366 } 367 } 368 _, _, err1 = RawSyscall(sys_SETGID, uintptr(cred.Gid), 0, 0) 369 if err1 != 0 { 370 goto childerror 371 } 372 _, _, err1 = RawSyscall(sys_SETUID, uintptr(cred.Uid), 0, 0) 373 if err1 != 0 { 374 goto childerror 375 } 376 } 377 378 if len(sys.AmbientCaps) != 0 { 379 // Ambient capabilities were added in the 4.3 kernel, 380 // so it is safe to always use _LINUX_CAPABILITY_VERSION_3. 381 caps.hdr.version = _LINUX_CAPABILITY_VERSION_3 382 383 if _, _, err1 := RawSyscall(SYS_CAPGET, uintptr(unsafe.Pointer(&caps.hdr)), uintptr(unsafe.Pointer(&caps.data[0])), 0); err1 != 0 { 384 goto childerror 385 } 386 387 for _, c := range sys.AmbientCaps { 388 // Add the c capability to the permitted and inheritable capability mask, 389 // otherwise we will not be able to add it to the ambient capability mask. 390 caps.data[capToIndex(c)].permitted |= capToMask(c) 391 caps.data[capToIndex(c)].inheritable |= capToMask(c) 392 } 393 394 if _, _, err1 := RawSyscall(SYS_CAPSET, uintptr(unsafe.Pointer(&caps.hdr)), uintptr(unsafe.Pointer(&caps.data[0])), 0); err1 != 0 { 395 goto childerror 396 } 397 398 for _, c := range sys.AmbientCaps { 399 _, _, err1 = RawSyscall6(SYS_PRCTL, PR_CAP_AMBIENT, uintptr(PR_CAP_AMBIENT_RAISE), c, 0, 0, 0) 400 if err1 != 0 { 401 goto childerror 402 } 403 } 404 } 405 406 // Chdir 407 if dir != nil { 408 _, _, err1 = RawSyscall(SYS_CHDIR, uintptr(unsafe.Pointer(dir)), 0, 0) 409 if err1 != 0 { 410 goto childerror 411 } 412 } 413 414 // Parent death signal 415 if sys.Pdeathsig != 0 { 416 _, _, err1 = RawSyscall6(SYS_PRCTL, PR_SET_PDEATHSIG, uintptr(sys.Pdeathsig), 0, 0, 0, 0) 417 if err1 != 0 { 418 goto childerror 419 } 420 421 // Signal self if parent is already dead. This might cause a 422 // duplicate signal in rare cases, but it won't matter when 423 // using SIGKILL. 424 r1, _ = rawSyscallNoError(SYS_GETPPID, 0, 0, 0) 425 if r1 != ppid { 426 pid, _ := rawSyscallNoError(SYS_GETPID, 0, 0, 0) 427 _, _, err1 := RawSyscall(SYS_KILL, pid, uintptr(sys.Pdeathsig), 0) 428 if err1 != 0 { 429 goto childerror 430 } 431 } 432 } 433 434 // Pass 1: look for fd[i] < i and move those up above len(fd) 435 // so that pass 2 won't stomp on an fd it needs later. 436 if pipe < nextfd { 437 _, _, err1 = RawSyscall(_SYS_dup, uintptr(pipe), uintptr(nextfd), 0) 438 if err1 != 0 { 439 goto childerror 440 } 441 RawSyscall(fcntl64Syscall, uintptr(nextfd), F_SETFD, FD_CLOEXEC) 442 pipe = nextfd 443 nextfd++ 444 } 445 for i = 0; i < len(fd); i++ { 446 if fd[i] >= 0 && fd[i] < int(i) { 447 if nextfd == pipe { // don't stomp on pipe 448 nextfd++ 449 } 450 _, _, err1 = RawSyscall(_SYS_dup, uintptr(fd[i]), uintptr(nextfd), 0) 451 if err1 != 0 { 452 goto childerror 453 } 454 RawSyscall(fcntl64Syscall, uintptr(nextfd), F_SETFD, FD_CLOEXEC) 455 fd[i] = nextfd 456 nextfd++ 457 } 458 } 459 460 // Pass 2: dup fd[i] down onto i. 461 for i = 0; i < len(fd); i++ { 462 if fd[i] == -1 { 463 RawSyscall(SYS_CLOSE, uintptr(i), 0, 0) 464 continue 465 } 466 if fd[i] == int(i) { 467 // dup2(i, i) won't clear close-on-exec flag on Linux, 468 // probably not elsewhere either. 469 _, _, err1 = RawSyscall(fcntl64Syscall, uintptr(fd[i]), F_SETFD, 0) 470 if err1 != 0 { 471 goto childerror 472 } 473 continue 474 } 475 // The new fd is created NOT close-on-exec, 476 // which is exactly what we want. 477 _, _, err1 = RawSyscall(_SYS_dup, uintptr(fd[i]), uintptr(i), 0) 478 if err1 != 0 { 479 goto childerror 480 } 481 } 482 483 // By convention, we don't close-on-exec the fds we are 484 // started with, so if len(fd) < 3, close 0, 1, 2 as needed. 485 // Programs that know they inherit fds >= 3 will need 486 // to set them close-on-exec. 487 for i = len(fd); i < 3; i++ { 488 RawSyscall(SYS_CLOSE, uintptr(i), 0, 0) 489 } 490 491 // Detach fd 0 from tty 492 if sys.Noctty { 493 _, _, err1 = RawSyscall(SYS_IOCTL, 0, uintptr(TIOCNOTTY), 0) 494 if err1 != 0 { 495 goto childerror 496 } 497 } 498 499 // Set the controlling TTY to Ctty 500 if sys.Setctty { 501 _, _, err1 = RawSyscall(SYS_IOCTL, uintptr(sys.Ctty), uintptr(TIOCSCTTY), 1) 502 if err1 != 0 { 503 goto childerror 504 } 505 } 506 507 // Enable tracing if requested. 508 // Do this right before exec so that we don't unnecessarily trace the runtime 509 // setting up after the fork. See issue #21428. 510 if sys.Ptrace { 511 _, _, err1 = RawSyscall(SYS_PTRACE, uintptr(PTRACE_TRACEME), 0, 0) 512 if err1 != 0 { 513 goto childerror 514 } 515 } 516 517 // Time to exec. 518 _, _, err1 = RawSyscall(SYS_EXECVE, 519 uintptr(unsafe.Pointer(argv0)), 520 uintptr(unsafe.Pointer(&argv[0])), 521 uintptr(unsafe.Pointer(&envv[0]))) 522 523childerror: 524 // send error code on pipe 525 RawSyscall(SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err1)), unsafe.Sizeof(err1)) 526 for { 527 RawSyscall(SYS_EXIT, 253, 0, 0) 528 } 529} 530 531// Try to open a pipe with O_CLOEXEC set on both file descriptors. 532func forkExecPipe(p []int) (err error) { 533 err = Pipe2(p, O_CLOEXEC) 534 // pipe2 was added in 2.6.27 and our minimum requirement is 2.6.23, so it 535 // might not be implemented. 536 if err == ENOSYS { 537 if err = Pipe(p); err != nil { 538 return 539 } 540 if _, err = fcntl(p[0], F_SETFD, FD_CLOEXEC); err != nil { 541 return 542 } 543 _, err = fcntl(p[1], F_SETFD, FD_CLOEXEC) 544 } 545 return 546} 547 548func formatIDMappings(idMap []SysProcIDMap) []byte { 549 var data []byte 550 for _, im := range idMap { 551 data = append(data, []byte(itoa(im.ContainerID)+" "+itoa(im.HostID)+" "+itoa(im.Size)+"\n")...) 552 } 553 return data 554} 555 556// writeIDMappings writes the user namespace User ID or Group ID mappings to the specified path. 557func writeIDMappings(path string, idMap []SysProcIDMap) error { 558 fd, err := Open(path, O_RDWR, 0) 559 if err != nil { 560 return err 561 } 562 563 if _, err := Write(fd, formatIDMappings(idMap)); err != nil { 564 Close(fd) 565 return err 566 } 567 568 if err := Close(fd); err != nil { 569 return err 570 } 571 572 return nil 573} 574 575// writeSetgroups writes to /proc/PID/setgroups "deny" if enable is false 576// and "allow" if enable is true. 577// This is needed since kernel 3.19, because you can't write gid_map without 578// disabling setgroups() system call. 579func writeSetgroups(pid int, enable bool) error { 580 sgf := "/proc/" + itoa(pid) + "/setgroups" 581 fd, err := Open(sgf, O_RDWR, 0) 582 if err != nil { 583 return err 584 } 585 586 var data []byte 587 if enable { 588 data = []byte("allow") 589 } else { 590 data = []byte("deny") 591 } 592 593 if _, err := Write(fd, data); err != nil { 594 Close(fd) 595 return err 596 } 597 598 return Close(fd) 599} 600 601// writeUidGidMappings writes User ID and Group ID mappings for user namespaces 602// for a process and it is called from the parent process. 603func writeUidGidMappings(pid int, sys *SysProcAttr) error { 604 if sys.UidMappings != nil { 605 uidf := "/proc/" + itoa(pid) + "/uid_map" 606 if err := writeIDMappings(uidf, sys.UidMappings); err != nil { 607 return err 608 } 609 } 610 611 if sys.GidMappings != nil { 612 // If the kernel is too old to support /proc/PID/setgroups, writeSetGroups will return ENOENT; this is OK. 613 if err := writeSetgroups(pid, sys.GidMappingsEnableSetgroups); err != nil && err != ENOENT { 614 return err 615 } 616 gidf := "/proc/" + itoa(pid) + "/gid_map" 617 if err := writeIDMappings(gidf, sys.GidMappings); err != nil { 618 return err 619 } 620 } 621 622 return nil 623} 624