1// Copyright 2020 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package poll
6
7import (
8	"internal/syscall/unix"
9	"sync/atomic"
10	"syscall"
11)
12
13var copyFileRangeSupported int32 = -1 // accessed atomically
14
15const maxCopyFileRangeRound = 1 << 30
16
17func kernelVersion() (major int, minor int) {
18	var uname syscall.Utsname
19	if err := syscall.Uname(&uname); err != nil {
20		return
21	}
22
23	rl := uname.Release
24	var values [2]int
25	vi := 0
26	value := 0
27	for _, c := range rl {
28		if '0' <= c && c <= '9' {
29			value = (value * 10) + int(c-'0')
30		} else {
31			// Note that we're assuming N.N.N here.  If we see anything else we are likely to
32			// mis-parse it.
33			values[vi] = value
34			vi++
35			if vi >= len(values) {
36				break
37			}
38			value = 0
39		}
40	}
41	switch vi {
42	case 0:
43		return 0, 0
44	case 1:
45		return values[0], 0
46	case 2:
47		return values[0], values[1]
48	}
49	return
50}
51
52// CopyFileRange copies at most remain bytes of data from src to dst, using
53// the copy_file_range system call. dst and src must refer to regular files.
54func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err error) {
55	if supported := atomic.LoadInt32(&copyFileRangeSupported); supported == 0 {
56		return 0, false, nil
57	} else if supported == -1 {
58		major, minor := kernelVersion()
59		if major > 5 || (major == 5 && minor >= 3) {
60			atomic.StoreInt32(&copyFileRangeSupported, 1)
61		} else {
62			// copy_file_range(2) is broken in various ways on kernels older than 5.3,
63			// see issue #42400 and
64			// https://man7.org/linux/man-pages/man2/copy_file_range.2.html#VERSIONS
65			atomic.StoreInt32(&copyFileRangeSupported, 0)
66			return 0, false, nil
67		}
68	}
69	for remain > 0 {
70		max := remain
71		if max > maxCopyFileRangeRound {
72			max = maxCopyFileRangeRound
73		}
74		n, err := copyFileRange(dst, src, int(max))
75		switch err {
76		case syscall.ENOSYS:
77			// copy_file_range(2) was introduced in Linux 4.5.
78			// Go supports Linux >= 2.6.33, so the system call
79			// may not be present.
80			//
81			// If we see ENOSYS, we have certainly not transfered
82			// any data, so we can tell the caller that we
83			// couldn't handle the transfer and let them fall
84			// back to more generic code.
85			//
86			// Seeing ENOSYS also means that we will not try to
87			// use copy_file_range(2) again.
88			atomic.StoreInt32(&copyFileRangeSupported, 0)
89			return 0, false, nil
90		case syscall.EXDEV, syscall.EINVAL, syscall.EIO, syscall.EOPNOTSUPP, syscall.EPERM:
91			// Prior to Linux 5.3, it was not possible to
92			// copy_file_range across file systems. Similarly to
93			// the ENOSYS case above, if we see EXDEV, we have
94			// not transfered any data, and we can let the caller
95			// fall back to generic code.
96			//
97			// As for EINVAL, that is what we see if, for example,
98			// dst or src refer to a pipe rather than a regular
99			// file. This is another case where no data has been
100			// transfered, so we consider it unhandled.
101			//
102			// If src and dst are on CIFS, we can see EIO.
103			// See issue #42334.
104			//
105			// If the file is on NFS, we can see EOPNOTSUPP.
106			// See issue #40731.
107			//
108			// If the process is running inside a Docker container,
109			// we might see EPERM instead of ENOSYS. See issue
110			// #40893. Since EPERM might also be a legitimate error,
111			// don't mark copy_file_range(2) as unsupported.
112			return 0, false, nil
113		case nil:
114			if n == 0 {
115				// If we did not read any bytes at all,
116				// then this file may be in a file system
117				// where copy_file_range silently fails.
118				// https://lore.kernel.org/linux-fsdevel/20210126233840.GG4626@dread.disaster.area/T/#m05753578c7f7882f6e9ffe01f981bc223edef2b0
119				if written == 0 {
120					return 0, false, nil
121				}
122				// Otherwise src is at EOF, which means
123				// we are done.
124				return written, true, nil
125			}
126			remain -= n
127			written += n
128		default:
129			return written, true, err
130		}
131	}
132	return written, true, nil
133}
134
135// copyFileRange performs one round of copy_file_range(2).
136func copyFileRange(dst, src *FD, max int) (written int64, err error) {
137	// The signature of copy_file_range(2) is:
138	//
139	// ssize_t copy_file_range(int fd_in, loff_t *off_in,
140	//                         int fd_out, loff_t *off_out,
141	//                         size_t len, unsigned int flags);
142	//
143	// Note that in the call to unix.CopyFileRange below, we use nil
144	// values for off_in and off_out. For the system call, this means
145	// "use and update the file offsets". That is why we must acquire
146	// locks for both file descriptors (and why this whole machinery is
147	// in the internal/poll package to begin with).
148	if err := dst.writeLock(); err != nil {
149		return 0, err
150	}
151	defer dst.writeUnlock()
152	if err := src.readLock(); err != nil {
153		return 0, err
154	}
155	defer src.readUnlock()
156	var n int
157	for {
158		n, err = unix.CopyFileRange(src.Sysfd, nil, dst.Sysfd, nil, max, 0)
159		if err != syscall.EINTR {
160			break
161		}
162	}
163	return int64(n), err
164}
165