1 // futex -*- C++ -*- 2 3 // Copyright (C) 2015-2022 Free Software Foundation, Inc. 4 // 5 // This file is part of the GNU ISO C++ Library. This library is free 6 // software; you can redistribute it and/or modify it under the 7 // terms of the GNU General Public License as published by the 8 // Free Software Foundation; either version 3, or (at your option) 9 // any later version. 10 11 // This library is distributed in the hope that it will be useful, 12 // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 // GNU General Public License for more details. 15 16 // Under Section 7 of GPL version 3, you are granted additional 17 // permissions described in the GCC Runtime Library Exception, version 18 // 3.1, as published by the Free Software Foundation. 19 20 // You should have received a copy of the GNU General Public License and 21 // a copy of the GCC Runtime Library Exception along with this program; 22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 23 // <http://www.gnu.org/licenses/>. 24 25 #include <bits/atomic_futex.h> 26 #ifdef _GLIBCXX_HAS_GTHREADS 27 #if defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1 28 #include <chrono> 29 #include <climits> 30 #include <syscall.h> 31 #include <unistd.h> 32 #include <sys/time.h> 33 #include <errno.h> 34 #include <ext/numeric_traits.h> 35 #include <debug/debug.h> 36 37 #ifdef _GLIBCXX_USE_CLOCK_GETTIME_SYSCALL 38 #include <unistd.h> 39 #include <sys/syscall.h> 40 #endif 41 42 // Constants for the wait/wake futex syscall operations 43 const unsigned futex_wait_op = 0; 44 const unsigned futex_wait_bitset_op = 9; 45 const unsigned futex_clock_monotonic_flag = 0; 46 const unsigned futex_clock_realtime_flag = 256; 47 const unsigned futex_bitset_match_any = ~0; 48 const unsigned futex_wake_op = 1; 49 50 namespace std _GLIBCXX_VISIBILITY(default) 51 { 52 _GLIBCXX_BEGIN_NAMESPACE_VERSION 53 54 using __gnu_cxx::__int_traits; 55 56 namespace 57 { 58 std::atomic<bool> futex_clock_realtime_unavailable; 59 std::atomic<bool> futex_clock_monotonic_unavailable; 60 61 #if defined(SYS_futex_time64) && SYS_futex_time64 != SYS_futex 62 // Userspace knows about the new time64 syscalls, so it's possible that 63 // userspace has also updated timespec to use a 64-bit tv_sec. 64 // The SYS_futex syscall still uses the old definition of timespec 65 // where tv_sec is 32 bits, so define a type that matches that. 66 struct syscall_timespec { long tv_sec; long tv_nsec; }; 67 using syscall_time_t = long; 68 #else 69 using syscall_timespec = ::timespec; 70 using syscall_time_t = time_t; 71 #endif 72 73 // Return the relative duration from (now_s + now_ns) to (abs_s + abs_ns) 74 // as a timespec suitable for syscalls. 75 syscall_timespec relative_timespec(chrono::seconds abs_s,chrono::nanoseconds abs_ns,time_t now_s,long now_ns)76 relative_timespec(chrono::seconds abs_s, chrono::nanoseconds abs_ns, 77 time_t now_s, long now_ns) 78 { 79 syscall_timespec rt; 80 81 // Did we already time out? 82 if (now_s > abs_s.count()) 83 { 84 rt.tv_sec = -1; 85 return rt; 86 } 87 88 const auto rel_s = abs_s.count() - now_s; 89 90 // Convert the absolute timeout to a relative timeout, without overflow. 91 if (rel_s > __int_traits<syscall_time_t>::__max) [[unlikely]] 92 { 93 rt.tv_sec = __int_traits<syscall_time_t>::__max; 94 rt.tv_nsec = 999999999; 95 } 96 else 97 { 98 rt.tv_sec = rel_s; 99 rt.tv_nsec = abs_ns.count() - now_ns; 100 if (rt.tv_nsec < 0) 101 { 102 rt.tv_nsec += 1000000000; 103 --rt.tv_sec; 104 } 105 } 106 107 return rt; 108 } 109 } // namespace 110 111 bool 112 __atomic_futex_unsigned_base:: _M_futex_wait_until(unsigned * __addr,unsigned __val,bool __has_timeout,chrono::seconds __s,chrono::nanoseconds __ns)113 _M_futex_wait_until(unsigned *__addr, unsigned __val, bool __has_timeout, 114 chrono::seconds __s, chrono::nanoseconds __ns) 115 { 116 if (!__has_timeout) 117 { 118 // Ignore whether we actually succeeded to block because at worst, 119 // we will fall back to spin-waiting. The only thing we could do 120 // here on errors is abort. 121 int ret __attribute__((unused)); 122 ret = syscall (SYS_futex, __addr, futex_wait_op, __val, nullptr); 123 __glibcxx_assert(ret == 0 || errno == EINTR || errno == EAGAIN); 124 return true; 125 } 126 else 127 { 128 if (!futex_clock_realtime_unavailable.load(std::memory_order_relaxed)) 129 { 130 // futex sets errno=EINVAL for absolute timeouts before the epoch. 131 if (__s.count() < 0) 132 return false; 133 134 syscall_timespec rt; 135 if (__s.count() > __int_traits<syscall_time_t>::__max) [[unlikely]] 136 rt.tv_sec = __int_traits<syscall_time_t>::__max; 137 else 138 rt.tv_sec = __s.count(); 139 rt.tv_nsec = __ns.count(); 140 141 if (syscall (SYS_futex, __addr, 142 futex_wait_bitset_op | futex_clock_realtime_flag, 143 __val, &rt, nullptr, futex_bitset_match_any) == -1) 144 { 145 __glibcxx_assert(errno == EINTR || errno == EAGAIN 146 || errno == ETIMEDOUT || errno == ENOSYS); 147 if (errno == ETIMEDOUT) 148 return false; 149 if (errno == ENOSYS) 150 { 151 futex_clock_realtime_unavailable.store(true, 152 std::memory_order_relaxed); 153 // Fall through to legacy implementation if the system 154 // call is unavailable. 155 } 156 else 157 return true; 158 } 159 else 160 return true; 161 } 162 163 // We only get to here if futex_clock_realtime_unavailable was 164 // true or has just been set to true. 165 struct timeval tv; 166 gettimeofday (&tv, NULL); 167 168 // Convert the absolute timeout value to a relative timeout 169 auto rt = relative_timespec(__s, __ns, tv.tv_sec, tv.tv_usec * 1000); 170 171 // Did we already time out? 172 if (rt.tv_sec < 0) 173 return false; 174 175 if (syscall (SYS_futex, __addr, futex_wait_op, __val, &rt) == -1) 176 { 177 __glibcxx_assert(errno == EINTR || errno == EAGAIN 178 || errno == ETIMEDOUT); 179 if (errno == ETIMEDOUT) 180 return false; 181 } 182 return true; 183 } 184 } 185 186 bool 187 __atomic_futex_unsigned_base:: _M_futex_wait_until_steady(unsigned * __addr,unsigned __val,bool __has_timeout,chrono::seconds __s,chrono::nanoseconds __ns)188 _M_futex_wait_until_steady(unsigned *__addr, unsigned __val, 189 bool __has_timeout, 190 chrono::seconds __s, chrono::nanoseconds __ns) 191 { 192 if (!__has_timeout) 193 { 194 // Ignore whether we actually succeeded to block because at worst, 195 // we will fall back to spin-waiting. The only thing we could do 196 // here on errors is abort. 197 int ret __attribute__((unused)); 198 ret = syscall (SYS_futex, __addr, futex_wait_op, __val, nullptr); 199 __glibcxx_assert(ret == 0 || errno == EINTR || errno == EAGAIN); 200 return true; 201 } 202 else 203 { 204 if (!futex_clock_monotonic_unavailable.load(std::memory_order_relaxed)) 205 { 206 // futex sets errno=EINVAL for absolute timeouts before the epoch. 207 if (__s.count() < 0) [[unlikely]] 208 return false; 209 210 syscall_timespec rt; 211 if (__s.count() > __int_traits<syscall_time_t>::__max) [[unlikely]] 212 rt.tv_sec = __int_traits<syscall_time_t>::__max; 213 else 214 rt.tv_sec = __s.count(); 215 rt.tv_nsec = __ns.count(); 216 217 if (syscall (SYS_futex, __addr, 218 futex_wait_bitset_op | futex_clock_monotonic_flag, 219 __val, &rt, nullptr, futex_bitset_match_any) == -1) 220 { 221 __glibcxx_assert(errno == EINTR || errno == EAGAIN 222 || errno == ETIMEDOUT || errno == ENOSYS); 223 if (errno == ETIMEDOUT) 224 return false; 225 else if (errno == ENOSYS) 226 { 227 futex_clock_monotonic_unavailable.store(true, 228 std::memory_order_relaxed); 229 // Fall through to legacy implementation if the system 230 // call is unavailable. 231 } 232 else 233 return true; 234 } 235 } 236 237 // We only get to here if futex_clock_monotonic_unavailable was 238 // true or has just been set to true. 239 struct timespec ts; 240 #ifdef _GLIBCXX_USE_CLOCK_GETTIME_SYSCALL 241 syscall(SYS_clock_gettime, CLOCK_MONOTONIC, &ts); 242 #else 243 clock_gettime(CLOCK_MONOTONIC, &ts); 244 #endif 245 246 // Convert the absolute timeout value to a relative timeout 247 auto rt = relative_timespec(__s, __ns, ts.tv_sec, ts.tv_nsec); 248 249 // Did we already time out? 250 if (rt.tv_sec < 0) 251 return false; 252 253 if (syscall (SYS_futex, __addr, futex_wait_op, __val, &rt) == -1) 254 { 255 __glibcxx_assert(errno == EINTR || errno == EAGAIN 256 || errno == ETIMEDOUT); 257 if (errno == ETIMEDOUT) 258 return false; 259 } 260 return true; 261 } 262 } 263 264 void _M_futex_notify_all(unsigned * __addr)265 __atomic_futex_unsigned_base::_M_futex_notify_all(unsigned* __addr) 266 { 267 // This syscall can fail for various reasons, including in situations 268 // in which there is no real error. Thus, we don't bother checking 269 // the error codes. See the futex documentation and glibc for background. 270 syscall (SYS_futex, __addr, futex_wake_op, INT_MAX); 271 } 272 273 _GLIBCXX_END_NAMESPACE_VERSION 274 } 275 #endif // defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1 276 #endif // _GLIBCXX_HAS_GTHREADS 277