1 // -*- C++ -*-
2 //===-- algorithm_impl.h --------------------------------------------------===//
3 //
4 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 // See https://llvm.org/LICENSE.txt for license information.
6 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //
8 //===----------------------------------------------------------------------===//
9
10 #ifndef _PSTL_ALGORITHM_IMPL_H
11 #define _PSTL_ALGORITHM_IMPL_H
12
13 #include <iterator>
14 #include <type_traits>
15 #include <utility>
16 #include <functional>
17 #include <algorithm>
18
19 #include "execution_impl.h"
20 #include "memory_impl.h"
21 #include "parallel_backend_utils.h"
22 #include "parallel_backend.h"
23 #include "parallel_impl.h"
24 #include "unseq_backend_simd.h"
25
26
27 namespace __pstl
28 {
29 namespace __internal
30 {
31
32 //------------------------------------------------------------------------
33 // any_of
34 //------------------------------------------------------------------------
35
36 template <class _ForwardIterator, class _Pred>
37 bool
__brick_any_of(const _ForwardIterator __first,const _ForwardIterator __last,_Pred __pred,std::false_type)38 __brick_any_of(const _ForwardIterator __first, const _ForwardIterator __last, _Pred __pred,
39 /*__is_vector=*/std::false_type) noexcept
40 {
41 return std::any_of(__first, __last, __pred);
42 };
43
44 template <class _ForwardIterator, class _Pred>
45 bool
__brick_any_of(const _ForwardIterator __first,const _ForwardIterator __last,_Pred __pred,std::true_type)46 __brick_any_of(const _ForwardIterator __first, const _ForwardIterator __last, _Pred __pred,
47 /*__is_vector=*/std::true_type) noexcept
48 {
49 return __unseq_backend::__simd_or(__first, __last - __first, __pred);
50 };
51
52 template <class _ExecutionPolicy, class _ForwardIterator, class _Pred, class _IsVector>
53 bool
__pattern_any_of(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Pred __pred,_IsVector __is_vector,std::false_type)54 __pattern_any_of(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred,
55 _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
56 {
57 return __internal::__brick_any_of(__first, __last, __pred, __is_vector);
58 }
59
60 template <class _ExecutionPolicy, class _ForwardIterator, class _Pred, class _IsVector>
61 bool
__pattern_any_of(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Pred __pred,_IsVector __is_vector,std::true_type)62 __pattern_any_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred,
63 _IsVector __is_vector, /*parallel=*/std::true_type)
64 {
65 return __internal::__except_handler([&]() {
66 return __internal::__parallel_or(std::forward<_ExecutionPolicy>(__exec), __first, __last,
67 [__pred, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
68 return __internal::__brick_any_of(__i, __j, __pred, __is_vector);
69 });
70 });
71 }
72
73 // [alg.foreach]
74 // for_each_n with no policy
75
76 template <class _ForwardIterator, class _Size, class _Function>
77 _ForwardIterator
__for_each_n_it_serial(_ForwardIterator __first,_Size __n,_Function __f)78 __for_each_n_it_serial(_ForwardIterator __first, _Size __n, _Function __f)
79 {
80 for (; __n > 0; ++__first, --__n)
81 __f(__first);
82 return __first;
83 }
84
85 //------------------------------------------------------------------------
86 // walk1 (pseudo)
87 //
88 // walk1 evaluates f(x) for each dereferenced value x drawn from [first,last)
89 //------------------------------------------------------------------------
90 template <class _ForwardIterator, class _Function>
91 void
__brick_walk1(_ForwardIterator __first,_ForwardIterator __last,_Function __f,std::false_type)92 __brick_walk1(_ForwardIterator __first, _ForwardIterator __last, _Function __f, /*vector=*/std::false_type) noexcept
93 {
94 std::for_each(__first, __last, __f);
95 }
96
97 template <class _RandomAccessIterator, class _Function>
98 void
__brick_walk1(_RandomAccessIterator __first,_RandomAccessIterator __last,_Function __f,std::true_type)99 __brick_walk1(_RandomAccessIterator __first, _RandomAccessIterator __last, _Function __f,
100 /*vector=*/std::true_type) noexcept
101 {
102 __unseq_backend::__simd_walk_1(__first, __last - __first, __f);
103 }
104
105 template <class _ExecutionPolicy, class _ForwardIterator, class _Function, class _IsVector>
106 void
__pattern_walk1(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Function __f,_IsVector __is_vector,std::false_type)107 __pattern_walk1(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Function __f,
108 _IsVector __is_vector,
109 /*parallel=*/std::false_type) noexcept
110 {
111 __internal::__brick_walk1(__first, __last, __f, __is_vector);
112 }
113
114 template <class _ExecutionPolicy, class _ForwardIterator, class _Function, class _IsVector>
115 void
__pattern_walk1(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Function __f,_IsVector __is_vector,std::true_type)116 __pattern_walk1(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Function __f,
117 _IsVector __is_vector,
118 /*parallel=*/std::true_type)
119 {
120 __internal::__except_handler([&]() {
121 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
122 [__f, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
123 __internal::__brick_walk1(__i, __j, __f, __is_vector);
124 });
125 });
126 }
127
128 template <class _ExecutionPolicy, class _ForwardIterator, class _Brick>
129 void
__pattern_walk_brick(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Brick __brick,std::false_type)130 __pattern_walk_brick(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Brick __brick,
131 /*parallel=*/std::false_type) noexcept
132 {
133 __brick(__first, __last);
134 }
135
136 template <class _ExecutionPolicy, class _ForwardIterator, class _Brick>
137 void
__pattern_walk_brick(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Brick __brick,std::true_type)138 __pattern_walk_brick(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Brick __brick,
139 /*parallel=*/std::true_type)
140 {
141 __internal::__except_handler([&]() {
142 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
143 [__brick](_ForwardIterator __i, _ForwardIterator __j) { __brick(__i, __j); });
144 });
145 }
146
147 //------------------------------------------------------------------------
148 // walk1_n
149 //------------------------------------------------------------------------
150 template <class _ForwardIterator, class _Size, class _Function>
151 _ForwardIterator
__brick_walk1_n(_ForwardIterator __first,_Size __n,_Function __f,std::false_type)152 __brick_walk1_n(_ForwardIterator __first, _Size __n, _Function __f, /*_IsVectorTag=*/std::false_type)
153 {
154 return __internal::__for_each_n_it_serial(__first, __n,
155 [&__f](_ForwardIterator __it) { __f(*__it); }); // calling serial version
156 }
157
158 template <class _RandomAccessIterator, class _DifferenceType, class _Function>
159 _RandomAccessIterator
__brick_walk1_n(_RandomAccessIterator __first,_DifferenceType __n,_Function __f,std::true_type)160 __brick_walk1_n(_RandomAccessIterator __first, _DifferenceType __n, _Function __f,
161 /*vectorTag=*/std::true_type) noexcept
162 {
163 return __unseq_backend::__simd_walk_1(__first, __n, __f);
164 }
165
166 template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function, class _IsVector>
167 _ForwardIterator
__pattern_walk1_n(_ExecutionPolicy &&,_ForwardIterator __first,_Size __n,_Function __f,_IsVector __is_vector,std::false_type)168 __pattern_walk1_n(_ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Function __f, _IsVector __is_vector,
169 /*is_parallel=*/std::false_type) noexcept
170 {
171 return __internal::__brick_walk1_n(__first, __n, __f, __is_vector);
172 }
173
174 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Function, class _IsVector>
175 _RandomAccessIterator
__pattern_walk1_n(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_Size __n,_Function __f,_IsVector __is_vector,std::true_type)176 __pattern_walk1_n(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _Size __n, _Function __f,
177 _IsVector __is_vector,
178 /*is_parallel=*/std::true_type)
179 {
180 __internal::__pattern_walk1(std::forward<_ExecutionPolicy>(__exec), __first, __first + __n, __f, __is_vector,
181 std::true_type());
182 return __first + __n;
183 }
184
185 template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Brick>
186 _ForwardIterator
__pattern_walk_brick_n(_ExecutionPolicy &&,_ForwardIterator __first,_Size __n,_Brick __brick,std::false_type)187 __pattern_walk_brick_n(_ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Brick __brick,
188 /*is_parallel=*/std::false_type) noexcept
189 {
190 return __brick(__first, __n);
191 }
192
193 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Brick>
194 _RandomAccessIterator
__pattern_walk_brick_n(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_Size __n,_Brick __brick,std::true_type)195 __pattern_walk_brick_n(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _Size __n, _Brick __brick,
196 /*is_parallel=*/std::true_type)
197 {
198 return __internal::__except_handler([&]() {
199 __par_backend::__parallel_for(
200 std::forward<_ExecutionPolicy>(__exec), __first, __first + __n,
201 [__brick](_RandomAccessIterator __i, _RandomAccessIterator __j) { __brick(__i, __j - __i); });
202 return __first + __n;
203 });
204 }
205
206 //------------------------------------------------------------------------
207 // walk2 (pseudo)
208 //
209 // walk2 evaluates f(x,y) for deferenced values (x,y) drawn from [first1,last1) and [first2,...)
210 //------------------------------------------------------------------------
211 template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
212 _ForwardIterator2
__brick_walk2(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_Function __f,std::false_type)213 __brick_walk2(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Function __f,
214 /*vector=*/std::false_type) noexcept
215 {
216 for (; __first1 != __last1; ++__first1, ++__first2)
217 __f(*__first1, *__first2);
218 return __first2;
219 }
220
221 template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
222 _ForwardIterator2
__brick_walk2(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_Function __f,std::true_type)223 __brick_walk2(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Function __f,
224 /*vector=*/std::true_type) noexcept
225 {
226 return __unseq_backend::__simd_walk_2(__first1, __last1 - __first1, __first2, __f);
227 }
228
229 template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
230 _ForwardIterator2
__brick_walk2_n(_ForwardIterator1 __first1,_Size __n,_ForwardIterator2 __first2,_Function __f,std::false_type)231 __brick_walk2_n(_ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2, _Function __f,
232 /*vector=*/std::false_type) noexcept
233 {
234 for (; __n > 0; --__n, ++__first1, ++__first2)
235 __f(*__first1, *__first2);
236 return __first2;
237 }
238
239 template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
240 _ForwardIterator2
__brick_walk2_n(_ForwardIterator1 __first1,_Size __n,_ForwardIterator2 __first2,_Function __f,std::true_type)241 __brick_walk2_n(_ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2, _Function __f,
242 /*vector=*/std::true_type) noexcept
243 {
244 return __unseq_backend::__simd_walk_2(__first1, __n, __first2, __f);
245 }
246
247 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function, class _IsVector>
248 _ForwardIterator2
__pattern_walk2(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_Function __f,_IsVector __is_vector,std::false_type)249 __pattern_walk2(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
250 _Function __f, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
251 {
252 return __internal::__brick_walk2(__first1, __last1, __first2, __f, __is_vector);
253 }
254
255 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function, class _IsVector>
256 _ForwardIterator2
__pattern_walk2(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_Function __f,_IsVector __is_vector,std::true_type)257 __pattern_walk2(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
258 _ForwardIterator2 __first2, _Function __f, _IsVector __is_vector, /*parallel=*/std::true_type)
259 {
260 return __internal::__except_handler([&]() {
261 __par_backend::__parallel_for(
262 std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
263 [__f, __first1, __first2, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
264 __internal::__brick_walk2(__i, __j, __first2 + (__i - __first1), __f, __is_vector);
265 });
266 return __first2 + (__last1 - __first1);
267 });
268 }
269
270 template <class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function,
271 class _IsVector>
272 _ForwardIterator2
__pattern_walk2_n(_ExecutionPolicy &&,_ForwardIterator1 __first1,_Size __n,_ForwardIterator2 __first2,_Function __f,_IsVector __is_vector,std::false_type)273 __pattern_walk2_n(_ExecutionPolicy&&, _ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2, _Function __f,
274 _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
275 {
276 return __internal::__brick_walk2_n(__first1, __n, __first2, __f, __is_vector);
277 }
278
279 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2,
280 class _Function, class _IsVector>
281 _RandomAccessIterator2
__pattern_walk2_n(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_Size __n,_RandomAccessIterator2 __first2,_Function __f,_IsVector __is_vector,std::true_type)282 __pattern_walk2_n(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _Size __n, _RandomAccessIterator2 __first2,
283 _Function __f, _IsVector __is_vector, /*parallel=*/std::true_type)
284 {
285 return __internal::__pattern_walk2(std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n, __first2, __f,
286 __is_vector, std::true_type());
287 }
288
289 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Brick>
290 _ForwardIterator2
__pattern_walk2_brick(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_Brick __brick,std::false_type)291 __pattern_walk2_brick(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
292 _ForwardIterator2 __first2, _Brick __brick, /*parallel=*/std::false_type) noexcept
293 {
294 return __brick(__first1, __last1, __first2);
295 }
296
297 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _Brick>
298 _RandomAccessIterator2
__pattern_walk2_brick(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_Brick __brick,std::true_type)299 __pattern_walk2_brick(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
300 _RandomAccessIterator2 __first2, _Brick __brick, /*parallel=*/std::true_type)
301 {
302 return __internal::__except_handler([&]() {
303 __par_backend::__parallel_for(
304 std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
305 [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
306 __brick(__i, __j, __first2 + (__i - __first1));
307 });
308 return __first2 + (__last1 - __first1);
309 });
310 }
311
312 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2, class _Brick>
313 _RandomAccessIterator2
__pattern_walk2_brick_n(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_Size __n,_RandomAccessIterator2 __first2,_Brick __brick,std::true_type)314 __pattern_walk2_brick_n(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _Size __n,
315 _RandomAccessIterator2 __first2, _Brick __brick, /*parallel=*/std::true_type)
316 {
317 return __internal::__except_handler([&]() {
318 __par_backend::__parallel_for(
319 std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
320 [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
321 __brick(__i, __j - __i, __first2 + (__i - __first1));
322 });
323 return __first2 + __n;
324 });
325 }
326
327 template <class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Brick>
328 _ForwardIterator2
__pattern_walk2_brick_n(_ExecutionPolicy &&,_ForwardIterator1 __first1,_Size __n,_ForwardIterator2 __first2,_Brick __brick,std::false_type)329 __pattern_walk2_brick_n(_ExecutionPolicy&&, _ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2,
330 _Brick __brick, /*parallel=*/std::false_type) noexcept
331 {
332 return __brick(__first1, __n, __first2);
333 }
334
335 //------------------------------------------------------------------------
336 // walk3 (pseudo)
337 //
338 // walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
339 //------------------------------------------------------------------------
340 template <class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3, class _Function>
341 _ForwardIterator3
__brick_walk3(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator3 __first3,_Function __f,std::false_type)342 __brick_walk3(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
343 _ForwardIterator3 __first3, _Function __f, /*vector=*/std::false_type) noexcept
344 {
345 for (; __first1 != __last1; ++__first1, ++__first2, ++__first3)
346 __f(*__first1, *__first2, *__first3);
347 return __first3;
348 }
349
350 template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Function>
351 _RandomAccessIterator3
__brick_walk3(_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator3 __first3,_Function __f,std::true_type)352 __brick_walk3(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
353 _RandomAccessIterator3 __first3, _Function __f, /*vector=*/std::true_type) noexcept
354 {
355 return __unseq_backend::__simd_walk_3(__first1, __last1 - __first1, __first2, __first3, __f);
356 }
357
358 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3,
359 class _Function, class _IsVector>
360 _ForwardIterator3
__pattern_walk3(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator3 __first3,_Function __f,_IsVector __is_vector,std::false_type)361 __pattern_walk3(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
362 _ForwardIterator3 __first3, _Function __f, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
363 {
364 return __internal::__brick_walk3(__first1, __last1, __first2, __first3, __f, __is_vector);
365 }
366
367 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
368 class _RandomAccessIterator3, class _Function, class _IsVector>
369 _RandomAccessIterator3
__pattern_walk3(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator3 __first3,_Function __f,_IsVector __is_vector,std::true_type)370 __pattern_walk3(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
371 _RandomAccessIterator2 __first2, _RandomAccessIterator3 __first3, _Function __f, _IsVector __is_vector,
372 /*parallel=*/std::true_type)
373 {
374 return __internal::__except_handler([&]() {
375 __par_backend::__parallel_for(
376 std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
377 [__f, __first1, __first2, __first3, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
378 __internal::__brick_walk3(__i, __j, __first2 + (__i - __first1), __first3 + (__i - __first1), __f,
379 __is_vector);
380 });
381 return __first3 + (__last1 - __first1);
382 });
383 }
384
385 //------------------------------------------------------------------------
386 // equal
387 //------------------------------------------------------------------------
388
389 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
390 bool
__brick_equal(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_BinaryPredicate __p,std::false_type)391 __brick_equal(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
392 _ForwardIterator2 __last2, _BinaryPredicate __p, /* IsVector = */ std::false_type) noexcept
393 {
394 return std::equal(__first1, __last1, __first2, __last2, __p);
395 }
396
397 template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
398 bool
__brick_equal(_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator2 __last2,_BinaryPredicate __p,std::true_type)399 __brick_equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
400 _RandomAccessIterator2 __last2, _BinaryPredicate __p, /* is_vector = */ std::true_type) noexcept
401 {
402 if (__last1 - __first1 != __last2 - __first2)
403 return false;
404
405 return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2, std::not_fn(__p)).first == __last1;
406 }
407
408 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
409 class _IsVector>
410 bool
__pattern_equal(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_BinaryPredicate __p,_IsVector __is_vector,std::false_type)411 __pattern_equal(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
412 _ForwardIterator2 __last2, _BinaryPredicate __p, _IsVector __is_vector, /* is_parallel = */
413 std::false_type) noexcept
414 {
415 return __internal::__brick_equal(__first1, __last1, __first2, __last2, __p, __is_vector);
416 }
417
418 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate,
419 class _IsVector>
420 bool
__pattern_equal(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator2 __last2,_BinaryPredicate __p,_IsVector __is_vector,std::true_type)421 __pattern_equal(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
422 _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2, _BinaryPredicate __p,
423 _IsVector __is_vector, /*is_parallel=*/std::true_type)
424 {
425 if (__last1 - __first1 != __last2 - __first2)
426 return false;
427
428 return __internal::__except_handler([&]() {
429 return !__internal::__parallel_or(
430 std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
431 [__first1, __first2, __p, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
432 return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1),
433 __p, __is_vector);
434 });
435 });
436 }
437
438 //------------------------------------------------------------------------
439 // equal version for sequences with equal length
440 //------------------------------------------------------------------------
441
442 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
443 bool
__brick_equal(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_BinaryPredicate __p,std::false_type)444 __brick_equal(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _BinaryPredicate __p,
445 /* IsVector = */ std::false_type) noexcept
446 {
447 return std::equal(__first1, __last1, __first2, __p);
448 }
449
450 template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
451 bool
__brick_equal(_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_BinaryPredicate __p,std::true_type)452 __brick_equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
453 _BinaryPredicate __p, /* is_vector = */ std::true_type) noexcept
454 {
455 return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2, std::not_fn(__p)).first == __last1;
456 }
457
458 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
459 class _IsVector>
460 bool
__pattern_equal(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_BinaryPredicate __p,_IsVector __is_vector,std::false_type)461 __pattern_equal(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
462 _BinaryPredicate __p, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept
463 {
464 return __internal::__brick_equal(__first1, __last1, __first2, __p, __is_vector);
465 }
466
467 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate,
468 class _IsVector>
469 bool
__pattern_equal(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_BinaryPredicate __p,_IsVector __is_vector,std::true_type)470 __pattern_equal(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
471 _RandomAccessIterator2 __first2, _BinaryPredicate __p, _IsVector __is_vector,
472 /*is_parallel=*/std::true_type)
473 {
474 return __internal::__except_handler([&]() {
475 return !__internal::__parallel_or(
476 std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
477 [__first1, __first2, __p, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
478 return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1), __p, __is_vector);
479 });
480 });
481 }
482
483 //------------------------------------------------------------------------
484 // find_if
485 //------------------------------------------------------------------------
486 template <class _ForwardIterator, class _Predicate>
487 _ForwardIterator
__brick_find_if(_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,std::false_type)488 __brick_find_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
489 /*is_vector=*/std::false_type) noexcept
490 {
491 return std::find_if(__first, __last, __pred);
492 }
493
494 template <class _RandomAccessIterator, class _Predicate>
495 _RandomAccessIterator
__brick_find_if(_RandomAccessIterator __first,_RandomAccessIterator __last,_Predicate __pred,std::true_type)496 __brick_find_if(_RandomAccessIterator __first, _RandomAccessIterator __last, _Predicate __pred,
497 /*is_vector=*/std::true_type) noexcept
498 {
499 typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
500 return __unseq_backend::__simd_first(
501 __first, _SizeType(0), __last - __first,
502 [&__pred](_RandomAccessIterator __it, _SizeType __i) { return __pred(__it[__i]); });
503 }
504
505 template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
506 _ForwardIterator
__pattern_find_if(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,_IsVector __is_vector,std::false_type)507 __pattern_find_if(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
508 _IsVector __is_vector,
509 /*is_parallel=*/std::false_type) noexcept
510 {
511 return __internal::__brick_find_if(__first, __last, __pred, __is_vector);
512 }
513
514 template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
515 _ForwardIterator
__pattern_find_if(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,_IsVector __is_vector,std::true_type)516 __pattern_find_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
517 _IsVector __is_vector,
518 /*is_parallel=*/std::true_type)
519 {
520 return __internal::__except_handler([&]() {
521 return __internal::__parallel_find(
522 std::forward<_ExecutionPolicy>(__exec), __first, __last,
523 [__pred, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
524 return __internal::__brick_find_if(__i, __j, __pred, __is_vector);
525 },
526 std::less<typename std::iterator_traits<_ForwardIterator>::difference_type>(),
527 /*is_first=*/true);
528 });
529 }
530
531 //------------------------------------------------------------------------
532 // find_end
533 //------------------------------------------------------------------------
534
535 // find the first occurrence of the subsequence [s_first, s_last)
536 // or the last occurrence of the subsequence in the range [first, last)
537 // b_first determines what occurrence we want to find (first or last)
538 template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate, class _IsVector>
539 _RandomAccessIterator1
__find_subrange(_RandomAccessIterator1 __first,_RandomAccessIterator1 __last,_RandomAccessIterator1 __global_last,_RandomAccessIterator2 __s_first,_RandomAccessIterator2 __s_last,_BinaryPredicate __pred,bool __b_first,_IsVector __is_vector)540 __find_subrange(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator1 __global_last,
541 _RandomAccessIterator2 __s_first, _RandomAccessIterator2 __s_last, _BinaryPredicate __pred,
542 bool __b_first, _IsVector __is_vector) noexcept
543 {
544 typedef typename std::iterator_traits<_RandomAccessIterator2>::value_type _ValueType;
545 auto __n2 = __s_last - __s_first;
546 if (__n2 < 1)
547 {
548 return __b_first ? __first : __last;
549 }
550
551 auto __n1 = __global_last - __first;
552 if (__n1 < __n2)
553 {
554 return __last;
555 }
556
557 auto __cur = __last;
558 while (__first != __last && (__global_last - __first >= __n2))
559 {
560 // find position of *s_first in [first, last) (it can be start of subsequence)
561 __first = __internal::__brick_find_if(
562 __first, __last, __equal_value_by_pred<_ValueType, _BinaryPredicate>(*__s_first, __pred), __is_vector);
563
564 // if position that was found previously is the start of subsequence
565 // then we can exit the loop (b_first == true) or keep the position
566 // (b_first == false)
567 if (__first != __last && (__global_last - __first >= __n2) &&
568 __internal::__brick_equal(__s_first + 1, __s_last, __first + 1, __pred, __is_vector))
569 {
570 if (__b_first)
571 {
572 return __first;
573 }
574 else
575 {
576 __cur = __first;
577 }
578 }
579 else if (__first == __last)
580 {
581 break;
582 }
583 else
584 {
585 }
586
587 // in case of b_first == false we try to find new start position
588 // for the next subsequence
589 ++__first;
590 }
591 return __cur;
592 }
593
594 template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate, class _IsVector>
595 _RandomAccessIterator
__find_subrange(_RandomAccessIterator __first,_RandomAccessIterator __last,_RandomAccessIterator __global_last,_Size __count,const _Tp & __value,_BinaryPredicate __pred,_IsVector __is_vector)596 __find_subrange(_RandomAccessIterator __first, _RandomAccessIterator __last, _RandomAccessIterator __global_last,
597 _Size __count, const _Tp& __value, _BinaryPredicate __pred, _IsVector __is_vector) noexcept
598 {
599 if (static_cast<_Size>(__global_last - __first) < __count || __count < 1)
600 {
601 return __last; // According to the standard last shall be returned when count < 1
602 }
603
604 auto __unary_pred = __equal_value_by_pred<_Tp, _BinaryPredicate>(__value, __pred);
605 while (__first != __last && (static_cast<_Size>(__global_last - __first) >= __count))
606 {
607 __first = __internal::__brick_find_if(__first, __last, __unary_pred, __is_vector);
608
609 // check that all of elements in [first+1, first+count) equal to value
610 if (__first != __last && (static_cast<_Size>(__global_last - __first) >= __count) &&
611 !__internal::__brick_any_of(__first + 1, __first + __count, std::not_fn(__unary_pred), __is_vector))
612 {
613 return __first;
614 }
615 else if (__first == __last)
616 {
617 break;
618 }
619 else
620 {
621 ++__first;
622 }
623 }
624 return __last;
625 }
626
627 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
628 _ForwardIterator1
__brick_find_end(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::false_type)629 __brick_find_end(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
630 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::false_type) noexcept
631 {
632 return std::find_end(__first, __last, __s_first, __s_last, __pred);
633 }
634
635 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
636 _ForwardIterator1
__brick_find_end(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::true_type)637 __brick_find_end(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
638 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::true_type) noexcept
639 {
640 return __find_subrange(__first, __last, __last, __s_first, __s_last, __pred, false, std::true_type());
641 }
642
643 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
644 class _IsVector>
645 _ForwardIterator1
__pattern_find_end(_ExecutionPolicy &&,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)646 __pattern_find_end(_ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
647 _ForwardIterator2 __s_last, _BinaryPredicate __pred, _IsVector __is_vector,
648 /*is_parallel=*/std::false_type) noexcept
649 {
650 return __internal::__brick_find_end(__first, __last, __s_first, __s_last, __pred, __is_vector);
651 }
652
653 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
654 class _IsVector>
655 _ForwardIterator1
__pattern_find_end(_ExecutionPolicy && __exec,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)656 __pattern_find_end(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
657 _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
658 _IsVector __is_vector, /*is_parallel=*/std::true_type) noexcept
659 {
660 if (__last - __first == __s_last - __s_first)
661 {
662 const bool __res = __internal::__pattern_equal(std::forward<_ExecutionPolicy>(__exec), __first, __last,
663 __s_first, __pred, __is_vector, std::true_type());
664 return __res ? __first : __last;
665 }
666 else
667 {
668 return __internal::__except_handler([&]() {
669 return __internal::__parallel_find(
670 std::forward<_ExecutionPolicy>(__exec), __first, __last,
671 [__last, __s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
672 return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, false,
673 __is_vector);
674 },
675 std::greater<typename std::iterator_traits<_ForwardIterator1>::difference_type>(), /*is_first=*/false);
676 });
677 }
678 }
679
680 //------------------------------------------------------------------------
681 // find_first_of
682 //------------------------------------------------------------------------
683 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
684 _ForwardIterator1
__brick_find_first_of(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::false_type)685 __brick_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
686 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::false_type) noexcept
687 {
688 return std::find_first_of(__first, __last, __s_first, __s_last, __pred);
689 }
690
691 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
692 _ForwardIterator1
__brick_find_first_of(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::true_type)693 __brick_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
694 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::true_type) noexcept
695 {
696 return __unseq_backend::__simd_find_first_of(__first, __last, __s_first, __s_last, __pred);
697 }
698
699 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
700 class _IsVector>
701 _ForwardIterator1
__pattern_find_first_of(_ExecutionPolicy &&,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)702 __pattern_find_first_of(_ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last,
703 _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
704 _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
705 {
706 return __internal::__brick_find_first_of(__first, __last, __s_first, __s_last, __pred, __is_vector);
707 }
708
709 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
710 class _IsVector>
711 _ForwardIterator1
__pattern_find_first_of(_ExecutionPolicy && __exec,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)712 __pattern_find_first_of(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
713 _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
714 _IsVector __is_vector, /*is_parallel=*/std::true_type) noexcept
715 {
716 return __internal::__except_handler([&]() {
717 return __internal::__parallel_find(
718 std::forward<_ExecutionPolicy>(__exec), __first, __last,
719 [__s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
720 return __internal::__brick_find_first_of(__i, __j, __s_first, __s_last, __pred, __is_vector);
721 },
722 std::less<typename std::iterator_traits<_ForwardIterator1>::difference_type>(), /*is_first=*/true);
723 });
724 }
725
726 //------------------------------------------------------------------------
727 // search
728 //------------------------------------------------------------------------
729 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
730 _ForwardIterator1
__brick_search(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::false_type)731 __brick_search(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
732 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
733 {
734 return std::search(__first, __last, __s_first, __s_last, __pred);
735 }
736
737 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
738 _ForwardIterator1
__brick_search(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::true_type)739 __brick_search(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
740 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
741 {
742 return __internal::__find_subrange(__first, __last, __last, __s_first, __s_last, __pred, true, std::true_type());
743 }
744
745 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
746 class _IsVector>
747 _ForwardIterator1
__pattern_search(_ExecutionPolicy &&,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)748 __pattern_search(_ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
749 _ForwardIterator2 __s_last, _BinaryPredicate __pred, _IsVector __is_vector,
750 /*is_parallel=*/std::false_type) noexcept
751 {
752 return __internal::__brick_search(__first, __last, __s_first, __s_last, __pred, __is_vector);
753 }
754
755 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
756 class _IsVector>
757 _ForwardIterator1
__pattern_search(_ExecutionPolicy && __exec,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)758 __pattern_search(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
759 _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
760 _IsVector __is_vector,
761 /*is_parallel=*/std::true_type) noexcept
762 {
763 if (__last - __first == __s_last - __s_first)
764 {
765 const bool __res = __internal::__pattern_equal(std::forward<_ExecutionPolicy>(__exec), __first, __last,
766 __s_first, __pred, __is_vector, std::true_type());
767 return __res ? __first : __last;
768 }
769 else
770 {
771 return __internal::__except_handler([&]() {
772 return __internal::__parallel_find(
773 std::forward<_ExecutionPolicy>(__exec), __first, __last,
774 [__last, __s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
775 return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, true,
776 __is_vector);
777 },
778 std::less<typename std::iterator_traits<_ForwardIterator1>::difference_type>(), /*is_first=*/true);
779 });
780 }
781 }
782
783 //------------------------------------------------------------------------
784 // search_n
785 //------------------------------------------------------------------------
786 template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
787 _ForwardIterator
__brick_search_n(_ForwardIterator __first,_ForwardIterator __last,_Size __count,const _Tp & __value,_BinaryPredicate __pred,std::false_type)788 __brick_search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value,
789 _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
790 {
791 return std::search_n(__first, __last, __count, __value, __pred);
792 }
793
794 template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
795 _ForwardIterator
__brick_search_n(_ForwardIterator __first,_ForwardIterator __last,_Size __count,const _Tp & __value,_BinaryPredicate __pred,std::true_type)796 __brick_search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value,
797 _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
798 {
799 return __internal::__find_subrange(__first, __last, __last, __count, __value, __pred, std::true_type());
800 }
801
802 template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate,
803 class _IsVector>
804 _ForwardIterator
__pattern_search_n(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Size __count,const _Tp & __value,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)805 __pattern_search_n(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
806 const _Tp& __value, _BinaryPredicate __pred, _IsVector __is_vector,
807 /*is_parallel=*/std::false_type) noexcept
808 {
809 return __internal::__brick_search_n(__first, __last, __count, __value, __pred, __is_vector);
810 }
811
812 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate,
813 class _IsVector>
814 _RandomAccessIterator
__pattern_search_n(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_Size __count,const _Tp & __value,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)815 __pattern_search_n(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
816 _Size __count, const _Tp& __value, _BinaryPredicate __pred, _IsVector __is_vector,
817 /*is_parallel=*/std::true_type) noexcept
818 {
819 if (static_cast<_Size>(__last - __first) == __count)
820 {
821 const bool __result = !__internal::__pattern_any_of(
822 std::forward<_ExecutionPolicy>(__exec), __first, __last,
823 [&__value, &__pred](const _Tp& __val) { return !__pred(__val, __value); }, __is_vector,
824 /*is_parallel*/ std::true_type());
825 return __result ? __first : __last;
826 }
827 else
828 {
829 return __internal::__except_handler([&__exec, __first, __last, __count, &__value, __pred, __is_vector]() {
830 return __internal::__parallel_find(
831 std::forward<_ExecutionPolicy>(__exec), __first, __last,
832 [__last, __count, &__value, __pred, __is_vector](_RandomAccessIterator __i, _RandomAccessIterator __j) {
833 return __internal::__find_subrange(__i, __j, __last, __count, __value, __pred, __is_vector);
834 },
835 std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(), /*is_first=*/true);
836 });
837 }
838 }
839
840 //------------------------------------------------------------------------
841 // copy_n
842 //------------------------------------------------------------------------
843
844 template <class _ForwardIterator, class _Size, class _OutputIterator>
845 _OutputIterator
__brick_copy_n(_ForwardIterator __first,_Size __n,_OutputIterator __result,std::false_type)846 __brick_copy_n(_ForwardIterator __first, _Size __n, _OutputIterator __result, /*vector=*/std::false_type) noexcept
847 {
848 return std::copy_n(__first, __n, __result);
849 }
850
851 template <class _ForwardIterator, class _Size, class _OutputIterator>
852 _OutputIterator
__brick_copy_n(_ForwardIterator __first,_Size __n,_OutputIterator __result,std::true_type)853 __brick_copy_n(_ForwardIterator __first, _Size __n, _OutputIterator __result, /*vector=*/std::true_type) noexcept
854 {
855 return __unseq_backend::__simd_assign(
856 __first, __n, __result, [](_ForwardIterator __first, _OutputIterator __result) { *__result = *__first; });
857 }
858
859 //------------------------------------------------------------------------
860 // copy
861 //------------------------------------------------------------------------
862 template <class _ForwardIterator, class _OutputIterator>
863 _OutputIterator
__brick_copy(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,std::false_type)864 __brick_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
865 /*vector=*/std::false_type) noexcept
866 {
867 return std::copy(__first, __last, __result);
868 }
869
870 template <class _RandomAccessIterator, class _OutputIterator>
871 _OutputIterator
__brick_copy(_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator __result,std::true_type)872 __brick_copy(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
873 /*vector=*/std::true_type) noexcept
874 {
875 return __unseq_backend::__simd_assign(
876 __first, __last - __first, __result,
877 [](_RandomAccessIterator __first, _OutputIterator __result) { *__result = *__first; });
878 }
879
880 //------------------------------------------------------------------------
881 // move
882 //------------------------------------------------------------------------
883 template <class _ForwardIterator, class _OutputIterator>
884 _OutputIterator
__brick_move(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,std::false_type)885 __brick_move(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
886 /*vector=*/std::false_type) noexcept
887 {
888 return std::move(__first, __last, __result);
889 }
890
891 template <class _RandomAccessIterator, class _OutputIterator>
892 _OutputIterator
__brick_move(_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator __result,std::true_type)893 __brick_move(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
894 /*vector=*/std::true_type) noexcept
895 {
896 return __unseq_backend::__simd_assign(
897 __first, __last - __first, __result,
898 [](_RandomAccessIterator __first, _OutputIterator __result) { *__result = std::move(*__first); });
899 }
900
901 struct __brick_move_destroy
902 {
903 template <typename _Iterator, typename _OutputIterator>
904 _OutputIterator
operator__brick_move_destroy905 operator()(_Iterator __first, _Iterator __last, _OutputIterator __result, /*vec*/ std::true_type) const
906 {
907 using _IteratorValueType = typename std::iterator_traits<_Iterator>::value_type;
908
909 return __unseq_backend::__simd_assign(__first, __last - __first, __result,
910 [](_Iterator __first, _OutputIterator __result) {
911 *__result = std::move(*__first);
912 (*__first).~_IteratorValueType();
913 });
914 }
915
916 template <typename _Iterator, typename _OutputIterator>
917 _OutputIterator
operator__brick_move_destroy918 operator()(_Iterator __first, _Iterator __last, _OutputIterator __result, /*vec*/ std::false_type) const
919 {
920 using _IteratorValueType = typename std::iterator_traits<_Iterator>::value_type;
921
922 for (; __first != __last; ++__first, ++__result)
923 {
924 *__result = std::move(*__first);
925 (*__first).~_IteratorValueType();
926 }
927 return __result;
928 }
929 };
930
931 //------------------------------------------------------------------------
932 // swap_ranges
933 //------------------------------------------------------------------------
934 template <class _ForwardIterator, class _OutputIterator>
935 _OutputIterator
__brick_swap_ranges(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,std::false_type)936 __brick_swap_ranges(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
937 /*vector=*/std::false_type) noexcept
938 {
939 return std::swap_ranges(__first, __last, __result);
940 }
941
942 template <class _ForwardIterator, class _OutputIterator>
943 _OutputIterator
__brick_swap_ranges(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,std::true_type)944 __brick_swap_ranges(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
945 /*vector=*/std::true_type) noexcept
946 {
947 using std::iter_swap;
948 return __unseq_backend::__simd_assign(__first, __last - __first, __result,
949 iter_swap<_ForwardIterator, _OutputIterator>);
950 }
951
952 //------------------------------------------------------------------------
953 // copy_if
954 //------------------------------------------------------------------------
955 template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
956 _OutputIterator
__brick_copy_if(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,_UnaryPredicate __pred,std::false_type)957 __brick_copy_if(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, _UnaryPredicate __pred,
958 /*vector=*/std::false_type) noexcept
959 {
960 return std::copy_if(__first, __last, __result, __pred);
961 }
962
963 template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
964 _OutputIterator
__brick_copy_if(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,_UnaryPredicate __pred,std::true_type)965 __brick_copy_if(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, _UnaryPredicate __pred,
966 /*vector=*/std::true_type) noexcept
967 {
968 #if (_PSTL_MONOTONIC_PRESENT)
969 return __unseq_backend::__simd_copy_if(__first, __last - __first, __result, __pred);
970 #else
971 return std::copy_if(__first, __last, __result, __pred);
972 #endif
973 }
974
975 // TODO: Try to use transform_reduce for combining __brick_copy_if_phase1 on IsVector.
976 template <class _DifferenceType, class _ForwardIterator, class _UnaryPredicate>
977 std::pair<_DifferenceType, _DifferenceType>
__brick_calc_mask_1(_ForwardIterator __first,_ForwardIterator __last,bool * __restrict __mask,_UnaryPredicate __pred,std::false_type)978 __brick_calc_mask_1(_ForwardIterator __first, _ForwardIterator __last, bool* __restrict __mask, _UnaryPredicate __pred,
979 /*vector=*/std::false_type) noexcept
980 {
981 auto __count_true = _DifferenceType(0);
982 auto __size = __last - __first;
983
984 static_assert(__is_random_access_iterator<_ForwardIterator>::value,
985 "Pattern-brick error. Should be a random access iterator.");
986
987 for (; __first != __last; ++__first, ++__mask)
988 {
989 *__mask = __pred(*__first);
990 if (*__mask)
991 {
992 ++__count_true;
993 }
994 }
995 return std::make_pair(__count_true, __size - __count_true);
996 }
997
998 template <class _DifferenceType, class _RandomAccessIterator, class _UnaryPredicate>
999 std::pair<_DifferenceType, _DifferenceType>
__brick_calc_mask_1(_RandomAccessIterator __first,_RandomAccessIterator __last,bool * __mask,_UnaryPredicate __pred,std::true_type)1000 __brick_calc_mask_1(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __mask, _UnaryPredicate __pred,
1001 /*vector=*/std::true_type) noexcept
1002 {
1003 auto __result = __unseq_backend::__simd_calc_mask_1(__first, __last - __first, __mask, __pred);
1004 return std::make_pair(__result, (__last - __first) - __result);
1005 }
1006
1007 template <class _ForwardIterator, class _OutputIterator, class _Assigner>
1008 void
__brick_copy_by_mask(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,bool * __mask,_Assigner __assigner,std::false_type)1009 __brick_copy_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, bool* __mask,
1010 _Assigner __assigner, /*vector=*/std::false_type) noexcept
1011 {
1012 for (; __first != __last; ++__first, ++__mask)
1013 {
1014 if (*__mask)
1015 {
1016 __assigner(__first, __result);
1017 ++__result;
1018 }
1019 }
1020 }
1021
1022 template <class _ForwardIterator, class _OutputIterator, class _Assigner>
1023 void
__brick_copy_by_mask(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,bool * __restrict __mask,_Assigner __assigner,std::true_type)1024 __brick_copy_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
1025 bool* __restrict __mask, _Assigner __assigner, /*vector=*/std::true_type) noexcept
1026 {
1027 #if (_PSTL_MONOTONIC_PRESENT)
1028 __unseq_backend::__simd_copy_by_mask(__first, __last - __first, __result, __mask, __assigner);
1029 #else
1030 __internal::__brick_copy_by_mask(__first, __last, __result, __mask, __assigner, std::false_type());
1031 #endif
1032 }
1033
1034 template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2>
1035 void
__brick_partition_by_mask(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,bool * __mask,std::false_type)1036 __brick_partition_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
1037 _OutputIterator2 __out_false, bool* __mask, /*vector=*/std::false_type) noexcept
1038 {
1039 for (; __first != __last; ++__first, ++__mask)
1040 {
1041 if (*__mask)
1042 {
1043 *__out_true = *__first;
1044 ++__out_true;
1045 }
1046 else
1047 {
1048 *__out_false = *__first;
1049 ++__out_false;
1050 }
1051 }
1052 }
1053
1054 template <class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2>
1055 void
__brick_partition_by_mask(_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,bool * __mask,std::true_type)1056 __brick_partition_by_mask(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator1 __out_true,
1057 _OutputIterator2 __out_false, bool* __mask, /*vector=*/std::true_type) noexcept
1058 {
1059 #if (_PSTL_MONOTONIC_PRESENT)
1060 __unseq_backend::__simd_partition_by_mask(__first, __last - __first, __out_true, __out_false, __mask);
1061 #else
1062 __internal::__brick_partition_by_mask(__first, __last, __out_true, __out_false, __mask, std::false_type());
1063 #endif
1064 }
1065
1066 template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryPredicate, class _IsVector>
1067 _OutputIterator
__pattern_copy_if(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)1068 __pattern_copy_if(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
1069 _UnaryPredicate __pred, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
1070 {
1071 return __internal::__brick_copy_if(__first, __last, __result, __pred, __is_vector);
1072 }
1073
1074 template <class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator, class _UnaryPredicate,
1075 class _IsVector>
1076 _OutputIterator
__pattern_copy_if(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator __result,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)1077 __pattern_copy_if(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
1078 _OutputIterator __result, _UnaryPredicate __pred, _IsVector __is_vector, /*parallel=*/std::true_type)
1079 {
1080 typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
1081 const _DifferenceType __n = __last - __first;
1082 if (_DifferenceType(1) < __n)
1083 {
1084 __par_backend::__buffer<bool> __mask_buf(__n);
1085 return __internal::__except_handler([&__exec, __n, __first, __result, __is_vector, __pred, &__mask_buf]() {
1086 bool* __mask = __mask_buf.get();
1087 _DifferenceType __m{};
1088 __par_backend::__parallel_strict_scan(
1089 std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
1090 [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
1091 return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len),
1092 __mask + __i, __pred, __is_vector)
1093 .first;
1094 },
1095 std::plus<_DifferenceType>(), // Combine
1096 [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
1097 __internal::__brick_copy_by_mask(
1098 __first + __i, __first + (__i + __len), __result + __initial, __mask + __i,
1099 [](_RandomAccessIterator __x, _OutputIterator __z) { *__z = *__x; }, __is_vector);
1100 },
1101 [&__m](_DifferenceType __total) { __m = __total; });
1102 return __result + __m;
1103 });
1104 }
1105 // trivial sequence - use serial algorithm
1106 return __internal::__brick_copy_if(__first, __last, __result, __pred, __is_vector);
1107 }
1108
1109 //------------------------------------------------------------------------
1110 // count
1111 //------------------------------------------------------------------------
1112 template <class _ForwardIterator, class _Predicate>
1113 typename std::iterator_traits<_ForwardIterator>::difference_type
__brick_count(_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,std::true_type)1114 __brick_count(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
1115 /* is_vector = */ std::true_type) noexcept
1116 {
1117 return __unseq_backend::__simd_count(__first, __last - __first, __pred);
1118 }
1119
1120 template <class _ForwardIterator, class _Predicate>
1121 typename std::iterator_traits<_ForwardIterator>::difference_type
__brick_count(_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,std::false_type)1122 __brick_count(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
1123 /* is_vector = */ std::false_type) noexcept
1124 {
1125 return std::count_if(__first, __last, __pred);
1126 }
1127
1128 template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
1129 typename std::iterator_traits<_ForwardIterator>::difference_type
__pattern_count(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,std::false_type,_IsVector __is_vector)1130 __pattern_count(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
1131 /* is_parallel */ std::false_type, _IsVector __is_vector) noexcept
1132 {
1133 return __internal::__brick_count(__first, __last, __pred, __is_vector);
1134 }
1135
1136 template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
1137 typename std::iterator_traits<_ForwardIterator>::difference_type
__pattern_count(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,std::true_type,_IsVector __is_vector)1138 __pattern_count(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
1139 /* is_parallel */ std::true_type, _IsVector __is_vector)
1140 {
1141 typedef typename std::iterator_traits<_ForwardIterator>::difference_type _SizeType;
1142 return __internal::__except_handler([&]() {
1143 return __par_backend::__parallel_reduce(
1144 std::forward<_ExecutionPolicy>(__exec), __first, __last, _SizeType(0),
1145 [__pred, __is_vector](_ForwardIterator __begin, _ForwardIterator __end, _SizeType __value) -> _SizeType {
1146 return __value + __internal::__brick_count(__begin, __end, __pred, __is_vector);
1147 },
1148 std::plus<_SizeType>());
1149 });
1150 }
1151
1152 //------------------------------------------------------------------------
1153 // unique
1154 //------------------------------------------------------------------------
1155
1156 template <class _ForwardIterator, class _BinaryPredicate>
1157 _ForwardIterator
__brick_unique(_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,std::false_type)1158 __brick_unique(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
1159 /*is_vector=*/std::false_type) noexcept
1160 {
1161 return std::unique(__first, __last, __pred);
1162 }
1163
1164 template <class _ForwardIterator, class _BinaryPredicate>
1165 _ForwardIterator
__brick_unique(_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,std::true_type)1166 __brick_unique(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
1167 /*is_vector=*/std::true_type) noexcept
1168 {
1169 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
1170 return std::unique(__first, __last, __pred);
1171 }
1172
1173 template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate, class _IsVector>
1174 _ForwardIterator
__pattern_unique(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)1175 __pattern_unique(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
1176 _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1177 {
1178 return __internal::__brick_unique(__first, __last, __pred, __is_vector);
1179 }
1180
1181 // That function is shared between two algorithms - remove_if (__pattern_remove_if) and unique (pattern unique). But a mask calculation is different.
1182 // So, a caller passes _CalcMask brick into remove_elements.
1183 template <class _ExecutionPolicy, class _ForwardIterator, class _CalcMask, class _IsVector>
1184 _ForwardIterator
__remove_elements(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_CalcMask __calc_mask,_IsVector __is_vector)1185 __remove_elements(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _CalcMask __calc_mask,
1186 _IsVector __is_vector)
1187 {
1188 typedef typename std::iterator_traits<_ForwardIterator>::difference_type _DifferenceType;
1189 typedef typename std::iterator_traits<_ForwardIterator>::value_type _Tp;
1190 _DifferenceType __n = __last - __first;
1191 __par_backend::__buffer<bool> __mask_buf(__n);
1192 // 1. find a first iterator that should be removed
1193 return __internal::__except_handler([&]() {
1194 bool* __mask = __mask_buf.get();
1195 _DifferenceType __min = __par_backend::__parallel_reduce(
1196 std::forward<_ExecutionPolicy>(__exec), _DifferenceType(0), __n, __n,
1197 [__first, __mask, &__calc_mask, __is_vector](_DifferenceType __i, _DifferenceType __j,
1198 _DifferenceType __local_min) -> _DifferenceType {
1199 // Create mask
1200 __calc_mask(__mask + __i, __mask + __j, __first + __i);
1201
1202 // if minimum was found in a previous range we shouldn't do anymore
1203 if (__local_min < __i)
1204 {
1205 return __local_min;
1206 }
1207 // find first iterator that should be removed
1208 bool* __result = __internal::__brick_find_if(__mask + __i, __mask + __j,
1209 [](bool __val) { return !__val; }, __is_vector);
1210 if (__result - __mask == __j)
1211 {
1212 return __local_min;
1213 }
1214 return std::min(__local_min, _DifferenceType(__result - __mask));
1215 },
1216 [](_DifferenceType __local_min1, _DifferenceType __local_min2) -> _DifferenceType {
1217 return std::min(__local_min1, __local_min2);
1218 });
1219
1220 // No elements to remove - exit
1221 if (__min == __n)
1222 {
1223 return __last;
1224 }
1225 __n -= __min;
1226 __first += __min;
1227
1228 __par_backend::__buffer<_Tp> __buf(__n);
1229 _Tp* __result = __buf.get();
1230 __mask += __min;
1231 _DifferenceType __m{};
1232 // 2. Elements that doesn't satisfy pred are moved to result
1233 __par_backend::__parallel_strict_scan(
1234 std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
1235 [__mask, __is_vector](_DifferenceType __i, _DifferenceType __len) {
1236 return __internal::__brick_count(__mask + __i, __mask + __i + __len, [](bool __val) { return __val; },
1237 __is_vector);
1238 },
1239 std::plus<_DifferenceType>(),
1240 [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) {
1241 __internal::__brick_copy_by_mask(
1242 __first + __i, __first + __i + __len, __result + __initial, __mask + __i,
1243 [](_ForwardIterator __x, _Tp* __z) {
1244 __internal::__invoke_if_else(std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); },
1245 [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); });
1246 },
1247 __is_vector);
1248 },
1249 [&__m](_DifferenceType __total) { __m = __total; });
1250
1251 // 3. Elements from result are moved to [first, last)
1252 __par_backend::__parallel_for(
1253 std::forward<_ExecutionPolicy>(__exec), __result, __result + __m,
1254 [__result, __first, __is_vector](_Tp* __i, _Tp* __j) {
1255 __invoke_if_else(
1256 std::is_trivial<_Tp>(),
1257 [&]() { __brick_move(__i, __j, __first + (__i - __result), __is_vector); },
1258 [&]() {
1259 __brick_move_destroy()(__i, __j, __first + (__i - __result), __is_vector);
1260 });
1261 });
1262 return __first + __m;
1263 });
1264 }
1265
1266 template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate, class _IsVector>
1267 _ForwardIterator
__pattern_unique(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)1268 __pattern_unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
1269 _IsVector __is_vector, /*is_parallel=*/std::true_type) noexcept
1270 {
1271 typedef typename std::iterator_traits<_ForwardIterator>::reference _ReferenceType;
1272
1273 if (__first == __last)
1274 {
1275 return __last;
1276 }
1277 if (__first + 1 == __last || __first + 2 == __last)
1278 {
1279 // Trivial sequence - use serial algorithm
1280 return __internal::__brick_unique(__first, __last, __pred, __is_vector);
1281 }
1282 return __internal::__remove_elements(
1283 std::forward<_ExecutionPolicy>(__exec), ++__first, __last,
1284 [&__pred, __is_vector](bool* __b, bool* __e, _ForwardIterator __it) {
1285 __internal::__brick_walk3(
1286 __b, __e, __it - 1, __it,
1287 [&__pred](bool& __x, _ReferenceType __y, _ReferenceType __z) { __x = !__pred(__y, __z); }, __is_vector);
1288 },
1289 __is_vector);
1290 }
1291
1292 //------------------------------------------------------------------------
1293 // unique_copy
1294 //------------------------------------------------------------------------
1295
1296 template <class _ForwardIterator, class OutputIterator, class _BinaryPredicate>
1297 OutputIterator
__brick_unique_copy(_ForwardIterator __first,_ForwardIterator __last,OutputIterator __result,_BinaryPredicate __pred,std::false_type)1298 __brick_unique_copy(_ForwardIterator __first, _ForwardIterator __last, OutputIterator __result, _BinaryPredicate __pred,
1299 /*vector=*/std::false_type) noexcept
1300 {
1301 return std::unique_copy(__first, __last, __result, __pred);
1302 }
1303
1304 template <class _RandomAccessIterator, class OutputIterator, class _BinaryPredicate>
1305 OutputIterator
__brick_unique_copy(_RandomAccessIterator __first,_RandomAccessIterator __last,OutputIterator __result,_BinaryPredicate __pred,std::true_type)1306 __brick_unique_copy(_RandomAccessIterator __first, _RandomAccessIterator __last, OutputIterator __result,
1307 _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
1308 {
1309 #if (_PSTL_MONOTONIC_PRESENT)
1310 return __unseq_backend::__simd_unique_copy(__first, __last - __first, __result, __pred);
1311 #else
1312 return std::unique_copy(__first, __last, __result, __pred);
1313 #endif
1314 }
1315
1316 template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryPredicate,
1317 class _IsVector>
1318 _OutputIterator
__pattern_unique_copy(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)1319 __pattern_unique_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
1320 _BinaryPredicate __pred, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
1321 {
1322 return __internal::__brick_unique_copy(__first, __last, __result, __pred, __is_vector);
1323 }
1324
1325 template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
1326 _DifferenceType
__brick_calc_mask_2(_RandomAccessIterator __first,_RandomAccessIterator __last,bool * __restrict __mask,_BinaryPredicate __pred,std::false_type)1327 __brick_calc_mask_2(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __restrict __mask,
1328 _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
1329 {
1330 _DifferenceType __count = 0;
1331 for (; __first != __last; ++__first, ++__mask)
1332 {
1333 *__mask = !__pred(*__first, *(__first - 1));
1334 __count += *__mask;
1335 }
1336 return __count;
1337 }
1338
1339 template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
1340 _DifferenceType
__brick_calc_mask_2(_RandomAccessIterator __first,_RandomAccessIterator __last,bool * __restrict __mask,_BinaryPredicate __pred,std::true_type)1341 __brick_calc_mask_2(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __restrict __mask,
1342 _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
1343 {
1344 return __unseq_backend::__simd_calc_mask_2(__first, __last - __first, __mask, __pred);
1345 }
1346
1347 template <class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator, class _BinaryPredicate,
1348 class _IsVector>
1349 _OutputIterator
__pattern_unique_copy(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator __result,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)1350 __pattern_unique_copy(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
1351 _OutputIterator __result, _BinaryPredicate __pred, _IsVector __is_vector,
1352 /*parallel=*/std::true_type)
1353 {
1354 typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
1355 const _DifferenceType __n = __last - __first;
1356 if (_DifferenceType(2) < __n)
1357 {
1358 __par_backend::__buffer<bool> __mask_buf(__n);
1359 if (_DifferenceType(2) < __n)
1360 {
1361 return __internal::__except_handler([&__exec, __n, __first, __result, __pred, __is_vector, &__mask_buf]() {
1362 bool* __mask = __mask_buf.get();
1363 _DifferenceType __m{};
1364 __par_backend::__parallel_strict_scan(
1365 std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
1366 [=](_DifferenceType __i, _DifferenceType __len) -> _DifferenceType { // Reduce
1367 _DifferenceType __extra = 0;
1368 if (__i == 0)
1369 {
1370 // Special boundary case
1371 __mask[__i] = true;
1372 if (--__len == 0)
1373 return 1;
1374 ++__i;
1375 ++__extra;
1376 }
1377 return __internal::__brick_calc_mask_2<_DifferenceType>(__first + __i, __first + (__i + __len),
1378 __mask + __i, __pred, __is_vector) +
1379 __extra;
1380 },
1381 std::plus<_DifferenceType>(), // Combine
1382 [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
1383 // Phase 2 is same as for __pattern_copy_if
1384 __internal::__brick_copy_by_mask(
1385 __first + __i, __first + (__i + __len), __result + __initial, __mask + __i,
1386 [](_RandomAccessIterator __x, _OutputIterator __z) { *__z = *__x; }, __is_vector);
1387 },
1388 [&__m](_DifferenceType __total) { __m = __total; });
1389 return __result + __m;
1390 });
1391 }
1392 }
1393 // trivial sequence - use serial algorithm
1394 return __internal::__brick_unique_copy(__first, __last, __result, __pred, __is_vector);
1395 }
1396
1397 //------------------------------------------------------------------------
1398 // reverse
1399 //------------------------------------------------------------------------
1400 template <class _BidirectionalIterator>
1401 void
__brick_reverse(_BidirectionalIterator __first,_BidirectionalIterator __last,std::false_type)1402 __brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, /*__is_vector=*/std::false_type) noexcept
1403 {
1404 std::reverse(__first, __last);
1405 }
1406
1407 template <class _BidirectionalIterator>
1408 void
__brick_reverse(_BidirectionalIterator __first,_BidirectionalIterator __last,std::true_type)1409 __brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, /*__is_vector=*/std::true_type) noexcept
1410 {
1411 typedef typename std::iterator_traits<_BidirectionalIterator>::reference _ReferenceType;
1412
1413 const auto __n = (__last - __first) / 2;
1414 __unseq_backend::__simd_walk_2(__first, __n, std::reverse_iterator<_BidirectionalIterator>(__last),
1415 [](_ReferenceType __x, _ReferenceType __y) {
1416 using std::swap;
1417 swap(__x, __y);
1418 });
1419 }
1420
1421 // this brick is called in parallel version, so we can use iterator arithmetic
1422 template <class _BidirectionalIterator>
1423 void
__brick_reverse(_BidirectionalIterator __first,_BidirectionalIterator __last,_BidirectionalIterator __d_last,std::false_type)1424 __brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, _BidirectionalIterator __d_last,
1425 /*is_vector=*/std::false_type) noexcept
1426 {
1427 for (--__d_last; __first != __last; ++__first, --__d_last)
1428 {
1429 using std::iter_swap;
1430 iter_swap(__first, __d_last);
1431 }
1432 }
1433
1434 // this brick is called in parallel version, so we can use iterator arithmetic
1435 template <class _BidirectionalIterator>
1436 void
__brick_reverse(_BidirectionalIterator __first,_BidirectionalIterator __last,_BidirectionalIterator __d_last,std::true_type)1437 __brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, _BidirectionalIterator __d_last,
1438 /*is_vector=*/std::true_type) noexcept
1439 {
1440 typedef typename std::iterator_traits<_BidirectionalIterator>::reference _ReferenceType;
1441
1442 __unseq_backend::__simd_walk_2(__first, __last - __first, std::reverse_iterator<_BidirectionalIterator>(__d_last),
1443 [](_ReferenceType __x, _ReferenceType __y) {
1444 using std::swap;
1445 swap(__x, __y);
1446 });
1447 }
1448
1449 template <class _ExecutionPolicy, class _BidirectionalIterator, class _IsVector>
1450 void
__pattern_reverse(_ExecutionPolicy &&,_BidirectionalIterator __first,_BidirectionalIterator __last,_IsVector _is_vector,std::false_type)1451 __pattern_reverse(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
1452 _IsVector _is_vector,
1453 /*is_parallel=*/std::false_type) noexcept
1454 {
1455 __internal::__brick_reverse(__first, __last, _is_vector);
1456 }
1457
1458 template <class _ExecutionPolicy, class _BidirectionalIterator, class _IsVector>
1459 void
__pattern_reverse(_ExecutionPolicy && __exec,_BidirectionalIterator __first,_BidirectionalIterator __last,_IsVector __is_vector,std::true_type)1460 __pattern_reverse(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
1461 _IsVector __is_vector, /*is_parallel=*/std::true_type)
1462 {
1463 __par_backend::__parallel_for(
1464 std::forward<_ExecutionPolicy>(__exec), __first, __first + (__last - __first) / 2,
1465 [__is_vector, __first, __last](_BidirectionalIterator __inner_first, _BidirectionalIterator __inner_last) {
1466 __internal::__brick_reverse(__inner_first, __inner_last, __last - (__inner_first - __first), __is_vector);
1467 });
1468 }
1469
1470 //------------------------------------------------------------------------
1471 // reverse_copy
1472 //------------------------------------------------------------------------
1473
1474 template <class _BidirectionalIterator, class _OutputIterator>
1475 _OutputIterator
__brick_reverse_copy(_BidirectionalIterator __first,_BidirectionalIterator __last,_OutputIterator __d_first,std::false_type)1476 __brick_reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __d_first,
1477 /*is_vector=*/std::false_type) noexcept
1478 {
1479 return std::reverse_copy(__first, __last, __d_first);
1480 }
1481
1482 template <class _BidirectionalIterator, class _OutputIterator>
1483 _OutputIterator
__brick_reverse_copy(_BidirectionalIterator __first,_BidirectionalIterator __last,_OutputIterator __d_first,std::true_type)1484 __brick_reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __d_first,
1485 /*is_vector=*/std::true_type) noexcept
1486 {
1487 typedef typename std::iterator_traits<_BidirectionalIterator>::reference _ReferenceType1;
1488 typedef typename std::iterator_traits<_OutputIterator>::reference _ReferenceType2;
1489
1490 return __unseq_backend::__simd_walk_2(std::reverse_iterator<_BidirectionalIterator>(__last), __last - __first,
1491 __d_first, [](_ReferenceType1 __x, _ReferenceType2 __y) { __y = __x; });
1492 }
1493
1494 template <class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator, class _IsVector>
1495 _OutputIterator
__pattern_reverse_copy(_ExecutionPolicy &&,_BidirectionalIterator __first,_BidirectionalIterator __last,_OutputIterator __d_first,_IsVector __is_vector,std::false_type)1496 __pattern_reverse_copy(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
1497 _OutputIterator __d_first, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1498 {
1499 return __internal::__brick_reverse_copy(__first, __last, __d_first, __is_vector);
1500 }
1501
1502 template <class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator, class _IsVector>
1503 _OutputIterator
__pattern_reverse_copy(_ExecutionPolicy && __exec,_BidirectionalIterator __first,_BidirectionalIterator __last,_OutputIterator __d_first,_IsVector __is_vector,std::true_type)1504 __pattern_reverse_copy(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
1505 _OutputIterator __d_first, _IsVector __is_vector, /*is_parallel=*/std::true_type)
1506 {
1507 auto __len = __last - __first;
1508 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
1509 [__is_vector, __first, __len, __d_first](_BidirectionalIterator __inner_first,
1510 _BidirectionalIterator __inner_last) {
1511 __internal::__brick_reverse_copy(__inner_first, __inner_last,
1512 __d_first + (__len - (__inner_last - __first)),
1513 __is_vector);
1514 });
1515 return __d_first + __len;
1516 }
1517
1518 //------------------------------------------------------------------------
1519 // rotate
1520 //------------------------------------------------------------------------
1521 template <class _ForwardIterator>
1522 _ForwardIterator
__brick_rotate(_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,std::false_type)1523 __brick_rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1524 /*is_vector=*/std::false_type) noexcept
1525 {
1526 #if _PSTL_CPP11_STD_ROTATE_BROKEN
1527 std::rotate(__first, __middle, __last);
1528 return std::next(__first, std::distance(__middle, __last));
1529 #else
1530 return std::rotate(__first, __middle, __last);
1531 #endif
1532 }
1533
1534 template <class _ForwardIterator>
1535 _ForwardIterator
__brick_rotate(_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,std::true_type)1536 __brick_rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1537 /*is_vector=*/std::true_type) noexcept
1538 {
1539 auto __n = __last - __first;
1540 auto __m = __middle - __first;
1541 const _ForwardIterator __ret = __first + (__last - __middle);
1542
1543 bool __is_left = (__m <= __n / 2);
1544 if (!__is_left)
1545 __m = __n - __m;
1546
1547 while (__n > 1 && __m > 0)
1548 {
1549 using std::iter_swap;
1550 const auto __m_2 = __m * 2;
1551 if (__is_left)
1552 {
1553 for (; __last - __first >= __m_2; __first += __m)
1554 {
1555 __unseq_backend::__simd_assign(__first, __m, __first + __m,
1556 iter_swap<_ForwardIterator, _ForwardIterator>);
1557 }
1558 }
1559 else
1560 {
1561 for (; __last - __first >= __m_2; __last -= __m)
1562 {
1563 __unseq_backend::__simd_assign(__last - __m, __m, __last - __m_2,
1564 iter_swap<_ForwardIterator, _ForwardIterator>);
1565 }
1566 }
1567 __is_left = !__is_left;
1568 __m = __n % __m;
1569 __n = __last - __first;
1570 }
1571
1572 return __ret;
1573 }
1574
1575 template <class _ExecutionPolicy, class _ForwardIterator, class _IsVector>
1576 _ForwardIterator
__pattern_rotate(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_IsVector __is_vector,std::false_type)1577 __pattern_rotate(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1578 _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1579 {
1580 return __internal::__brick_rotate(__first, __middle, __last, __is_vector);
1581 }
1582
1583 template <class _ExecutionPolicy, class _ForwardIterator, class _IsVector>
1584 _ForwardIterator
__pattern_rotate(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_IsVector __is_vector,std::true_type)1585 __pattern_rotate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __middle,
1586 _ForwardIterator __last, _IsVector __is_vector, /*is_parallel=*/std::true_type)
1587 {
1588 typedef typename std::iterator_traits<_ForwardIterator>::value_type _Tp;
1589 auto __n = __last - __first;
1590 auto __m = __middle - __first;
1591 if (__m <= __n / 2)
1592 {
1593 __par_backend::__buffer<_Tp> __buf(__n - __m);
1594 return __internal::__except_handler([&__exec, __n, __m, __first, __middle, __last, __is_vector, &__buf]() {
1595 _Tp* __result = __buf.get();
1596 __par_backend::__parallel_for(
1597 std::forward<_ExecutionPolicy>(__exec), __middle, __last,
1598 [__middle, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
1599 __internal::__brick_uninitialized_move(__b, __e, __result + (__b - __middle), __is_vector);
1600 });
1601
1602 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __middle,
1603 [__last, __middle, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
1604 __internal::__brick_move(__b, __e, __b + (__last - __middle),
1605 __is_vector);
1606 });
1607
1608 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __result, __result + (__n - __m),
1609 [__first, __result, __is_vector](_Tp* __b, _Tp* __e) {
1610 __brick_move_destroy()(
1611 __b, __e, __first + (__b - __result), __is_vector);
1612 });
1613
1614 return __first + (__last - __middle);
1615 });
1616 }
1617 else
1618 {
1619 __par_backend::__buffer<_Tp> __buf(__m);
1620 return __internal::__except_handler([&__exec, __n, __m, __first, __middle, __last, __is_vector, &__buf]() {
1621 _Tp* __result = __buf.get();
1622 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __middle,
1623 [__first, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
1624 __internal::__brick_uninitialized_move(
1625 __b, __e, __result + (__b - __first), __is_vector);
1626 });
1627
1628 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __middle, __last,
1629 [__first, __middle, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
1630 __internal::__brick_move(__b, __e, __first + (__b - __middle),
1631 __is_vector);
1632 });
1633
1634 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __result, __result + __m,
1635 [__n, __m, __first, __result, __is_vector](_Tp* __b, _Tp* __e) {
1636 __brick_move_destroy()(
1637 __b, __e, __first + ((__n - __m) + (__b - __result)), __is_vector);
1638 });
1639
1640 return __first + (__last - __middle);
1641 });
1642 }
1643 }
1644
1645 //------------------------------------------------------------------------
1646 // rotate_copy
1647 //------------------------------------------------------------------------
1648
1649 template <class _ForwardIterator, class _OutputIterator>
1650 _OutputIterator
__brick_rotate_copy(_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_OutputIterator __result,std::false_type)1651 __brick_rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1652 _OutputIterator __result, /*__is_vector=*/std::false_type) noexcept
1653 {
1654 return std::rotate_copy(__first, __middle, __last, __result);
1655 }
1656
1657 template <class _ForwardIterator, class _OutputIterator>
1658 _OutputIterator
__brick_rotate_copy(_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_OutputIterator __result,std::true_type)1659 __brick_rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1660 _OutputIterator __result, /*__is_vector=*/std::true_type) noexcept
1661 {
1662 _OutputIterator __res = __internal::__brick_copy(__middle, __last, __result, std::true_type());
1663 return __internal::__brick_copy(__first, __middle, __res, std::true_type());
1664 }
1665
1666 template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _IsVector>
1667 _OutputIterator
__pattern_rotate_copy(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_OutputIterator __result,_IsVector __is_vector,std::false_type)1668 __pattern_rotate_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1669 _OutputIterator __result, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1670 {
1671 return __internal::__brick_rotate_copy(__first, __middle, __last, __result, __is_vector);
1672 }
1673
1674 template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _IsVector>
1675 _OutputIterator
__pattern_rotate_copy(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_OutputIterator __result,_IsVector __is_vector,std::true_type)1676 __pattern_rotate_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __middle,
1677 _ForwardIterator __last, _OutputIterator __result, _IsVector __is_vector,
1678 /*is_parallel=*/std::true_type)
1679 {
1680 __par_backend::__parallel_for(
1681 std::forward<_ExecutionPolicy>(__exec), __first, __last,
1682 [__first, __last, __middle, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
1683 if (__b > __middle)
1684 {
1685 __internal::__brick_copy(__b, __e, __result + (__b - __middle), __is_vector);
1686 }
1687 else
1688 {
1689 _OutputIterator __new_result = __result + ((__last - __middle) + (__b - __first));
1690 if (__e < __middle)
1691 {
1692 __internal::__brick_copy(__b, __e, __new_result, __is_vector);
1693 }
1694 else
1695 {
1696 __internal::__brick_copy(__b, __middle, __new_result, __is_vector);
1697 __internal::__brick_copy(__middle, __e, __result, __is_vector);
1698 }
1699 }
1700 });
1701 return __result + (__last - __first);
1702 }
1703
1704 //------------------------------------------------------------------------
1705 // is_partitioned
1706 //------------------------------------------------------------------------
1707
1708 template <class _ForwardIterator, class _UnaryPredicate>
1709 bool
__brick_is_partitioned(_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,std::false_type)1710 __brick_is_partitioned(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1711 /*is_vector=*/std::false_type) noexcept
1712 {
1713 return std::is_partitioned(__first, __last, __pred);
1714 }
1715
1716 template <class _ForwardIterator, class _UnaryPredicate>
1717 bool
__brick_is_partitioned(_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,std::true_type)1718 __brick_is_partitioned(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1719 /*is_vector=*/std::true_type) noexcept
1720 {
1721 typedef typename std::iterator_traits<_ForwardIterator>::difference_type _SizeType;
1722 if (__first == __last)
1723 {
1724 return true;
1725 }
1726 else
1727 {
1728 _ForwardIterator __result = __unseq_backend::__simd_first(
1729 __first, _SizeType(0), __last - __first,
1730 [&__pred](_ForwardIterator __it, _SizeType __i) { return !__pred(__it[__i]); });
1731 if (__result == __last)
1732 {
1733 return true;
1734 }
1735 else
1736 {
1737 ++__result;
1738 return !__unseq_backend::__simd_or(__result, __last - __result, __pred);
1739 }
1740 }
1741 }
1742
1743 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
1744 bool
__pattern_is_partitioned(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)1745 __pattern_is_partitioned(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1746 _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1747 {
1748 return __internal::__brick_is_partitioned(__first, __last, __pred, __is_vector);
1749 }
1750
1751 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
1752 bool
__pattern_is_partitioned(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)1753 __pattern_is_partitioned(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
1754 _UnaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::true_type)
1755 {
1756 if (__first == __last)
1757 {
1758 return true;
1759 }
1760 else
1761 {
1762 return __internal::__except_handler([&]() {
1763 // State of current range:
1764 // broken - current range is not partitioned by pred
1765 // all_true - all elements in current range satisfy pred
1766 // all_false - all elements in current range don't satisfy pred
1767 // true_false - elements satisfy pred are placed before elements that don't satisfy pred
1768 enum _ReduceType
1769 {
1770 __not_init = -1,
1771 __broken,
1772 __all_true,
1773 __all_false,
1774 __true_false
1775 };
1776 _ReduceType __init = __not_init;
1777
1778 // Array with states that we'll have when state from the left branch is merged with state from the right branch.
1779 // State is calculated by formula: new_state = table[left_state * 4 + right_state]
1780 _ReduceType __table[] = {__broken, __broken, __broken, __broken, __broken, __all_true,
1781 __true_false, __true_false, __broken, __broken, __all_false, __broken,
1782 __broken, __broken, __true_false, __broken};
1783
1784 __init = __par_backend::__parallel_reduce(
1785 std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
1786 [&__pred, &__table, __is_vector](_ForwardIterator __i, _ForwardIterator __j,
1787 _ReduceType __value) -> _ReduceType {
1788 if (__value == __broken)
1789 {
1790 return __broken;
1791 }
1792 _ReduceType __res = __not_init;
1793 // if first element satisfy pred
1794 if (__pred(*__i))
1795 {
1796 // find first element that don't satisfy pred
1797 _ForwardIterator __x =
1798 __internal::__brick_find_if(__i + 1, __j, std::not_fn(__pred), __is_vector);
1799 if (__x != __j)
1800 {
1801 // find first element after "x" that satisfy pred
1802 _ForwardIterator __y = __internal::__brick_find_if(__x + 1, __j, __pred, __is_vector);
1803 // if it was found then range isn't partitioned by pred
1804 if (__y != __j)
1805 {
1806 return __broken;
1807 }
1808 else
1809 {
1810 __res = __true_false;
1811 }
1812 }
1813 else
1814 {
1815 __res = __all_true;
1816 }
1817 }
1818 else
1819 { // if first element doesn't satisfy pred
1820 // then we should find the first element that satisfy pred.
1821 // If we found it then range isn't partitioned by pred
1822 if (__internal::__brick_find_if(__i + 1, __j, __pred, __is_vector) != __j)
1823 {
1824 return __broken;
1825 }
1826 else
1827 {
1828 __res = __all_false;
1829 }
1830 }
1831 // if we have value from left range then we should calculate the result
1832 return (__value == -1) ? __res : __table[__value * 4 + __res];
1833 },
1834
1835 [&__table](_ReduceType __val1, _ReduceType __val2) -> _ReduceType {
1836 if (__val1 == __broken || __val2 == __broken)
1837 {
1838 return __broken;
1839 }
1840 // calculate the result for new big range
1841 return __table[__val1 * 4 + __val2];
1842 });
1843 return __init != __broken;
1844 });
1845 }
1846 }
1847
1848 //------------------------------------------------------------------------
1849 // partition
1850 //------------------------------------------------------------------------
1851
1852 template <class _ForwardIterator, class _UnaryPredicate>
1853 _ForwardIterator
__brick_partition(_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,std::false_type)1854 __brick_partition(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1855 /*is_vector=*/std::false_type) noexcept
1856 {
1857 return std::partition(__first, __last, __pred);
1858 }
1859
1860 template <class _ForwardIterator, class _UnaryPredicate>
1861 _ForwardIterator
__brick_partition(_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,std::true_type)1862 __brick_partition(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1863 /*is_vector=*/std::true_type) noexcept
1864 {
1865 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
1866 return std::partition(__first, __last, __pred);
1867 }
1868
1869 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
1870 _ForwardIterator
__pattern_partition(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)1871 __pattern_partition(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1872 _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1873 {
1874 return __internal::__brick_partition(__first, __last, __pred, __is_vector);
1875 }
1876
1877 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
1878 _ForwardIterator
__pattern_partition(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)1879 __pattern_partition(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
1880 _UnaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::true_type)
1881 {
1882
1883 // partitioned range: elements before pivot satisfy pred (true part),
1884 // elements after pivot don't satisfy pred (false part)
1885 struct _PartitionRange
1886 {
1887 _ForwardIterator __begin;
1888 _ForwardIterator __pivot;
1889 _ForwardIterator __end;
1890 };
1891
1892 return __internal::__except_handler([&]() {
1893 _PartitionRange __init{__last, __last, __last};
1894
1895 // lambda for merging two partitioned ranges to one partitioned range
1896 auto __reductor = [&__exec, __is_vector](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange {
1897 auto __size1 = __val1.__end - __val1.__pivot;
1898 auto __size2 = __val2.__pivot - __val2.__begin;
1899 auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
1900
1901 // if all elements in left range satisfy pred then we can move new pivot to pivot of right range
1902 if (__val1.__end == __val1.__pivot)
1903 {
1904 return {__new_begin, __val2.__pivot, __val2.__end};
1905 }
1906 // if true part of right range greater than false part of left range
1907 // then we should swap the false part of left range and last part of true part of right range
1908 else if (__size2 > __size1)
1909 {
1910 __par_backend::__parallel_for(
1911 std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size1,
1912 [__val1, __val2, __size1, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
1913 __internal::__brick_swap_ranges(__i, __j, (__val2.__pivot - __size1) + (__i - __val1.__pivot),
1914 __is_vector);
1915 });
1916 return {__new_begin, __val2.__pivot - __size1, __val2.__end};
1917 }
1918 // else we should swap the first part of false part of left range and true part of right range
1919 else
1920 {
1921 __par_backend::__parallel_for(
1922 std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size2,
1923 [__val1, __val2, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
1924 __internal::__brick_swap_ranges(__i, __j, __val2.__begin + (__i - __val1.__pivot), __is_vector);
1925 });
1926 return {__new_begin, __val1.__pivot + __size2, __val2.__end};
1927 }
1928 };
1929
1930 _PartitionRange __result = __par_backend::__parallel_reduce(
1931 std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
1932 [__pred, __is_vector, __reductor](_ForwardIterator __i, _ForwardIterator __j,
1933 _PartitionRange __value) -> _PartitionRange {
1934 //1. serial partition
1935 _ForwardIterator __pivot = __internal::__brick_partition(__i, __j, __pred, __is_vector);
1936
1937 // 2. merging of two ranges (left and right respectively)
1938 return __reductor(__value, {__i, __pivot, __j});
1939 },
1940 __reductor);
1941 return __result.__pivot;
1942 });
1943 }
1944
1945 //------------------------------------------------------------------------
1946 // stable_partition
1947 //------------------------------------------------------------------------
1948
1949 template <class _BidirectionalIterator, class _UnaryPredicate>
1950 _BidirectionalIterator
__brick_stable_partition(_BidirectionalIterator __first,_BidirectionalIterator __last,_UnaryPredicate __pred,std::false_type)1951 __brick_stable_partition(_BidirectionalIterator __first, _BidirectionalIterator __last, _UnaryPredicate __pred,
1952 /*__is_vector=*/std::false_type) noexcept
1953 {
1954 return std::stable_partition(__first, __last, __pred);
1955 }
1956
1957 template <class _BidirectionalIterator, class _UnaryPredicate>
1958 _BidirectionalIterator
__brick_stable_partition(_BidirectionalIterator __first,_BidirectionalIterator __last,_UnaryPredicate __pred,std::true_type)1959 __brick_stable_partition(_BidirectionalIterator __first, _BidirectionalIterator __last, _UnaryPredicate __pred,
1960 /*__is_vector=*/std::true_type) noexcept
1961 {
1962 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
1963 return std::stable_partition(__first, __last, __pred);
1964 }
1965
1966 template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate, class _IsVector>
1967 _BidirectionalIterator
__pattern_stable_partition(_ExecutionPolicy &&,_BidirectionalIterator __first,_BidirectionalIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)1968 __pattern_stable_partition(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
1969 _UnaryPredicate __pred, _IsVector __is_vector,
1970 /*is_parallelization=*/std::false_type) noexcept
1971 {
1972 return __internal::__brick_stable_partition(__first, __last, __pred, __is_vector);
1973 }
1974
1975 template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate, class _IsVector>
1976 _BidirectionalIterator
__pattern_stable_partition(_ExecutionPolicy && __exec,_BidirectionalIterator __first,_BidirectionalIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)1977 __pattern_stable_partition(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
1978 _UnaryPredicate __pred, _IsVector __is_vector,
1979 /*is_parallelization=*/std::true_type) noexcept
1980 {
1981 // partitioned range: elements before pivot satisfy pred (true part),
1982 // elements after pivot don't satisfy pred (false part)
1983 struct _PartitionRange
1984 {
1985 _BidirectionalIterator __begin;
1986 _BidirectionalIterator __pivot;
1987 _BidirectionalIterator __end;
1988 };
1989
1990 return __internal::__except_handler([&]() {
1991 _PartitionRange __init{__last, __last, __last};
1992
1993 // lambda for merging two partitioned ranges to one partitioned range
1994 auto __reductor = [__is_vector](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange {
1995 auto __size1 = __val1.__end - __val1.__pivot;
1996 auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
1997
1998 // if all elements in left range satisfy pred then we can move new pivot to pivot of right range
1999 if (__val1.__end == __val1.__pivot)
2000 {
2001 return {__new_begin, __val2.__pivot, __val2.__end};
2002 }
2003 // if true part of right range greater than false part of left range
2004 // then we should swap the false part of left range and last part of true part of right range
2005 else
2006 {
2007 __internal::__brick_rotate(__val1.__pivot, __val2.__begin, __val2.__pivot, __is_vector);
2008 return {__new_begin, __val2.__pivot - __size1, __val2.__end};
2009 }
2010 };
2011
2012 _PartitionRange __result = __par_backend::__parallel_reduce(
2013 std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
2014 [&__pred, __is_vector, __reductor](_BidirectionalIterator __i, _BidirectionalIterator __j,
2015 _PartitionRange __value) -> _PartitionRange {
2016 //1. serial stable_partition
2017 _BidirectionalIterator __pivot = __internal::__brick_stable_partition(__i, __j, __pred, __is_vector);
2018
2019 // 2. merging of two ranges (left and right respectively)
2020 return __reductor(__value, {__i, __pivot, __j});
2021 },
2022 __reductor);
2023 return __result.__pivot;
2024 });
2025 }
2026
2027 //------------------------------------------------------------------------
2028 // partition_copy
2029 //------------------------------------------------------------------------
2030
2031 template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
2032 std::pair<_OutputIterator1, _OutputIterator2>
__brick_partition_copy(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,_UnaryPredicate __pred,std::false_type)2033 __brick_partition_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
2034 _OutputIterator2 __out_false, _UnaryPredicate __pred, /*is_vector=*/std::false_type) noexcept
2035 {
2036 return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
2037 }
2038
2039 template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
2040 std::pair<_OutputIterator1, _OutputIterator2>
__brick_partition_copy(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,_UnaryPredicate __pred,std::true_type)2041 __brick_partition_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
2042 _OutputIterator2 __out_false, _UnaryPredicate __pred, /*is_vector=*/std::true_type) noexcept
2043 {
2044 #if (_PSTL_MONOTONIC_PRESENT)
2045 return __unseq_backend::__simd_partition_copy(__first, __last - __first, __out_true, __out_false, __pred);
2046 #else
2047 return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
2048 #endif
2049 }
2050
2051 template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator1, class _OutputIterator2,
2052 class _UnaryPredicate, class _IsVector>
2053 std::pair<_OutputIterator1, _OutputIterator2>
__pattern_partition_copy(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)2054 __pattern_partition_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
2055 _OutputIterator1 __out_true, _OutputIterator2 __out_false, _UnaryPredicate __pred,
2056 _IsVector __is_vector, /*is_parallelization=*/std::false_type) noexcept
2057 {
2058 return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred, __is_vector);
2059 }
2060
2061 template <class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2,
2062 class _UnaryPredicate, class _IsVector>
2063 std::pair<_OutputIterator1, _OutputIterator2>
__pattern_partition_copy(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)2064 __pattern_partition_copy(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
2065 _OutputIterator1 __out_true, _OutputIterator2 __out_false, _UnaryPredicate __pred,
2066 _IsVector __is_vector, /*is_parallelization=*/std::true_type)
2067 {
2068 typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
2069 typedef std::pair<_DifferenceType, _DifferenceType> _ReturnType;
2070 const _DifferenceType __n = __last - __first;
2071 if (_DifferenceType(1) < __n)
2072 {
2073 __par_backend::__buffer<bool> __mask_buf(__n);
2074 return __internal::__except_handler([&__exec, __n, __first, __out_true, __out_false, __is_vector, __pred,
2075 &__mask_buf]() {
2076 bool* __mask = __mask_buf.get();
2077 _ReturnType __m{};
2078 __par_backend::__parallel_strict_scan(
2079 std::forward<_ExecutionPolicy>(__exec), __n, std::make_pair(_DifferenceType(0), _DifferenceType(0)),
2080 [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
2081 return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len),
2082 __mask + __i, __pred, __is_vector);
2083 },
2084 [](const _ReturnType& __x, const _ReturnType& __y) -> _ReturnType {
2085 return std::make_pair(__x.first + __y.first, __x.second + __y.second);
2086 }, // Combine
2087 [=](_DifferenceType __i, _DifferenceType __len, _ReturnType __initial) { // Scan
2088 __internal::__brick_partition_by_mask(__first + __i, __first + (__i + __len),
2089 __out_true + __initial.first, __out_false + __initial.second,
2090 __mask + __i, __is_vector);
2091 },
2092 [&__m](_ReturnType __total) { __m = __total; });
2093 return std::make_pair(__out_true + __m.first, __out_false + __m.second);
2094 });
2095 }
2096 // trivial sequence - use serial algorithm
2097 return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred, __is_vector);
2098 }
2099
2100 //------------------------------------------------------------------------
2101 // sort
2102 //------------------------------------------------------------------------
2103
2104 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector,
2105 class _IsMoveConstructible>
2106 void
__pattern_sort(_ExecutionPolicy &&,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::false_type,_IsMoveConstructible)2107 __pattern_sort(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
2108 _IsVector /*is_vector*/, /*is_parallel=*/std::false_type, _IsMoveConstructible) noexcept
2109 {
2110 std::sort(__first, __last, __comp);
2111 }
2112
2113 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2114 void
__pattern_sort(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::true_type,std::true_type)2115 __pattern_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
2116 _IsVector /*is_vector*/, /*is_parallel=*/std::true_type, /*is_move_constructible=*/std::true_type)
2117 {
2118 __internal::__except_handler([&]() {
2119 __par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
2120 [](_RandomAccessIterator __first, _RandomAccessIterator __last,
2121 _Compare __comp) { std::sort(__first, __last, __comp); });
2122 });
2123 }
2124
2125 //------------------------------------------------------------------------
2126 // stable_sort
2127 //------------------------------------------------------------------------
2128
2129 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2130 void
__pattern_stable_sort(_ExecutionPolicy &&,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::false_type)2131 __pattern_stable_sort(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
2132 _IsVector /*is_vector*/, /*is_parallel=*/std::false_type) noexcept
2133 {
2134 std::stable_sort(__first, __last, __comp);
2135 }
2136
2137 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2138 void
__pattern_stable_sort(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::true_type)2139 __pattern_stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
2140 _Compare __comp, _IsVector /*is_vector*/, /*is_parallel=*/std::true_type)
2141 {
2142 __internal::__except_handler([&]() {
2143 __par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
2144 [](_RandomAccessIterator __first, _RandomAccessIterator __last,
2145 _Compare __comp) { std::stable_sort(__first, __last, __comp); });
2146 });
2147 }
2148
2149 //------------------------------------------------------------------------
2150 // partial_sort
2151 //------------------------------------------------------------------------
2152
2153 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2154 void
__pattern_partial_sort(_ExecutionPolicy &&,_RandomAccessIterator __first,_RandomAccessIterator __middle,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::false_type)2155 __pattern_partial_sort(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __middle,
2156 _RandomAccessIterator __last, _Compare __comp, _IsVector,
2157 /*is_parallel=*/std::false_type) noexcept
2158 {
2159 std::partial_sort(__first, __middle, __last, __comp);
2160 }
2161
2162 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2163 void
__pattern_partial_sort(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __middle,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::true_type)2164 __pattern_partial_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __middle,
2165 _RandomAccessIterator __last, _Compare __comp, _IsVector, /*is_parallel=*/std::true_type)
2166 {
2167 const auto __n = __middle - __first;
2168 if (__n == 0)
2169 return;
2170
2171 __internal::__except_handler([&]() {
2172 __par_backend::__parallel_stable_sort(
2173 std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
2174 [__n](_RandomAccessIterator __begin, _RandomAccessIterator __end, _Compare __comp) {
2175 if (__n < __end - __begin)
2176 std::partial_sort(__begin, __begin + __n, __end, __comp);
2177 else
2178 std::sort(__begin, __end, __comp);
2179 },
2180 __n);
2181 });
2182 }
2183
2184 //------------------------------------------------------------------------
2185 // partial_sort_copy
2186 //------------------------------------------------------------------------
2187
2188 template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare, class _IsVector>
2189 _RandomAccessIterator
__pattern_partial_sort_copy(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_RandomAccessIterator __d_first,_RandomAccessIterator __d_last,_Compare __comp,_IsVector,std::false_type)2190 __pattern_partial_sort_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
2191 _RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp, _IsVector,
2192 /*is_parallel=*/std::false_type) noexcept
2193 {
2194 return std::partial_sort_copy(__first, __last, __d_first, __d_last, __comp);
2195 }
2196
2197 template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare, class _IsVector>
2198 _RandomAccessIterator
__pattern_partial_sort_copy(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_RandomAccessIterator __d_first,_RandomAccessIterator __d_last,_Compare __comp,_IsVector __is_vector,std::true_type)2199 __pattern_partial_sort_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
2200 _RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp,
2201 _IsVector __is_vector, /*is_parallel=*/std::true_type)
2202 {
2203 if (__last == __first || __d_last == __d_first)
2204 {
2205 return __d_first;
2206 }
2207 auto __n1 = __last - __first;
2208 auto __n2 = __d_last - __d_first;
2209 return __internal::__except_handler([&]() {
2210 if (__n2 >= __n1)
2211 {
2212 __par_backend::__parallel_stable_sort(
2213 std::forward<_ExecutionPolicy>(__exec), __d_first, __d_first + __n1, __comp,
2214 [__first, __d_first, __is_vector](_RandomAccessIterator __i, _RandomAccessIterator __j,
2215 _Compare __comp) {
2216 _ForwardIterator __i1 = __first + (__i - __d_first);
2217 _ForwardIterator __j1 = __first + (__j - __d_first);
2218
2219 // 1. Copy elements from input to output
2220 # if !_PSTL_ICC_18_OMP_SIMD_BROKEN
2221 __internal::__brick_copy(__i1, __j1, __i, __is_vector);
2222 # else
2223 std::copy(__i1, __j1, __i);
2224 # endif
2225 // 2. Sort elements in output sequence
2226 std::sort(__i, __j, __comp);
2227 },
2228 __n1);
2229 return __d_first + __n1;
2230 }
2231 else
2232 {
2233 typedef typename std::iterator_traits<_ForwardIterator>::value_type _T1;
2234 typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _T2;
2235 __par_backend::__buffer<_T1> __buf(__n1);
2236 _T1* __r = __buf.get();
2237
2238 __par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __r, __r + __n1, __comp,
2239 [__n2, __first, __r](_T1* __i, _T1* __j, _Compare __comp) {
2240 _ForwardIterator __it = __first + (__i - __r);
2241
2242 // 1. Copy elements from input to raw memory
2243 for (_T1* __k = __i; __k != __j; ++__k, ++__it)
2244 {
2245 ::new (__k) _T2(*__it);
2246 }
2247
2248 // 2. Sort elements in temporary __buffer
2249 if (__n2 < __j - __i)
2250 std::partial_sort(__i, __i + __n2, __j, __comp);
2251 else
2252 std::sort(__i, __j, __comp);
2253 },
2254 __n2);
2255
2256 // 3. Move elements from temporary __buffer to output
2257 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __r, __r + __n2,
2258 [__r, __d_first, __is_vector](_T1* __i, _T1* __j) {
2259 __brick_move_destroy()(
2260 __i, __j, __d_first + (__i - __r), __is_vector);
2261 });
2262 __par_backend::__parallel_for(
2263 std::forward<_ExecutionPolicy>(__exec), __r + __n2, __r + __n1,
2264 [__is_vector](_T1* __i, _T1* __j) { __brick_destroy(__i, __j, __is_vector); });
2265
2266 return __d_first + __n2;
2267 }
2268 });
2269 }
2270
2271 //------------------------------------------------------------------------
2272 // adjacent_find
2273 //------------------------------------------------------------------------
2274 template <class _ForwardIterator, class _BinaryPredicate>
2275 _ForwardIterator
__brick_adjacent_find(_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,std::true_type,bool __or_semantic)2276 __brick_adjacent_find(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
2277 /* IsVector = */ std::true_type, bool __or_semantic) noexcept
2278 {
2279 return __unseq_backend::__simd_adjacent_find(__first, __last, __pred, __or_semantic);
2280 }
2281
2282 template <class _ForwardIterator, class _BinaryPredicate>
2283 _ForwardIterator
__brick_adjacent_find(_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,std::false_type,bool)2284 __brick_adjacent_find(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
2285 /* IsVector = */ std::false_type, bool) noexcept
2286 {
2287 return std::adjacent_find(__first, __last, __pred);
2288 }
2289
2290 template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate, class _IsVector>
2291 _ForwardIterator
__pattern_adjacent_find(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,std::false_type,_IsVector __is_vector,bool __or_semantic)2292 __pattern_adjacent_find(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
2293 /* is_parallel */ std::false_type, _IsVector __is_vector, bool __or_semantic) noexcept
2294 {
2295 return __internal::__brick_adjacent_find(__first, __last, __pred, __is_vector, __or_semantic);
2296 }
2297
2298 template <class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate, class _IsVector>
2299 _RandomAccessIterator
__pattern_adjacent_find(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_BinaryPredicate __pred,std::true_type,_IsVector __is_vector,bool __or_semantic)2300 __pattern_adjacent_find(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
2301 _BinaryPredicate __pred, /* is_parallel */ std::true_type, _IsVector __is_vector,
2302 bool __or_semantic)
2303 {
2304 if (__last - __first < 2)
2305 return __last;
2306
2307 return __internal::__except_handler([&]() {
2308 return __par_backend::__parallel_reduce(
2309 std::forward<_ExecutionPolicy>(__exec), __first, __last, __last,
2310 [__last, __pred, __is_vector, __or_semantic](_RandomAccessIterator __begin, _RandomAccessIterator __end,
2311 _RandomAccessIterator __value) -> _RandomAccessIterator {
2312 // TODO: investigate performance benefits from the use of shared variable for the result,
2313 // checking (compare_and_swap idiom) its __value at __first.
2314 if (__or_semantic && __value < __last)
2315 { //found
2316 __par_backend::__cancel_execution();
2317 return __value;
2318 }
2319
2320 if (__value > __begin)
2321 {
2322 // modify __end to check the predicate on the boundary __values;
2323 // TODO: to use a custom range with boundaries overlapping
2324 // TODO: investigate what if we remove "if" below and run algorithm on range [__first, __last-1)
2325 // then check the pair [__last-1, __last)
2326 if (__end != __last)
2327 ++__end;
2328
2329 //correct the global result iterator if the "brick" returns a local "__last"
2330 const _RandomAccessIterator __res =
2331 __internal::__brick_adjacent_find(__begin, __end, __pred, __is_vector, __or_semantic);
2332 if (__res < __end)
2333 __value = __res;
2334 }
2335 return __value;
2336 },
2337 [](_RandomAccessIterator __x, _RandomAccessIterator __y) -> _RandomAccessIterator {
2338 return __x < __y ? __x : __y;
2339 } //reduce a __value
2340 );
2341 });
2342 }
2343
2344 //------------------------------------------------------------------------
2345 // nth_element
2346 //------------------------------------------------------------------------
2347
2348 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2349 void
__pattern_nth_element(_ExecutionPolicy &&,_RandomAccessIterator __first,_RandomAccessIterator __nth,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::false_type)2350 __pattern_nth_element(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __nth,
2351 _RandomAccessIterator __last, _Compare __comp, _IsVector,
2352 /*is_parallel=*/std::false_type) noexcept
2353 {
2354 std::nth_element(__first, __nth, __last, __comp);
2355 }
2356
2357 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2358 void
__pattern_nth_element(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __nth,_RandomAccessIterator __last,_Compare __comp,_IsVector __is_vector,std::true_type)2359 __pattern_nth_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __nth,
2360 _RandomAccessIterator __last, _Compare __comp, _IsVector __is_vector,
2361 /*is_parallel=*/std::true_type) noexcept
2362 {
2363 if (__first == __last || __nth == __last)
2364 {
2365 return;
2366 }
2367
2368 using std::iter_swap;
2369 typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
2370 _RandomAccessIterator __x;
2371 do
2372 {
2373 __x = __internal::__pattern_partition(std::forward<_ExecutionPolicy>(__exec), __first + 1, __last,
2374 [&__comp, __first](const _Tp& __x) { return __comp(__x, *__first); },
2375 __is_vector,
2376 /*is_parallel=*/std::true_type());
2377 --__x;
2378 if (__x != __first)
2379 {
2380 iter_swap(__first, __x);
2381 }
2382 // if x > nth then our new range for partition is [first, x)
2383 if (__x - __nth > 0)
2384 {
2385 __last = __x;
2386 }
2387 // if x < nth then our new range for partition is [x, last)
2388 else if (__x - __nth < 0)
2389 {
2390 // if *x == *nth then we can start new partition with x+1
2391 if (!__comp(*__nth, *__x) && !__comp(*__x, *__nth))
2392 {
2393 ++__x;
2394 }
2395 else
2396 {
2397 iter_swap(__nth, __x);
2398 }
2399 __first = __x;
2400 }
2401 } while (__x != __nth);
2402 }
2403
2404 //------------------------------------------------------------------------
2405 // fill, fill_n
2406 //------------------------------------------------------------------------
2407 template <class _ForwardIterator, class _Tp>
2408 void
__brick_fill(_ForwardIterator __first,_ForwardIterator __last,const _Tp & __value,std::true_type)2409 __brick_fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value,
2410 /* __is_vector = */ std::true_type) noexcept
2411 {
2412 __unseq_backend::__simd_fill_n(__first, __last - __first, __value);
2413 }
2414
2415 template <class _ForwardIterator, class _Tp>
2416 void
__brick_fill(_ForwardIterator __first,_ForwardIterator __last,const _Tp & __value,std::false_type)2417 __brick_fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value,
2418 /* __is_vector = */ std::false_type) noexcept
2419 {
2420 std::fill(__first, __last, __value);
2421 }
2422
2423 template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _IsVector>
2424 void
__pattern_fill(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,const _Tp & __value,std::false_type,_IsVector __is_vector)2425 __pattern_fill(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value,
2426 /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept
2427 {
2428 __internal::__brick_fill(__first, __last, __value, __is_vector);
2429 }
2430
2431 template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _IsVector>
2432 _ForwardIterator
__pattern_fill(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,const _Tp & __value,std::true_type,_IsVector __is_vector)2433 __pattern_fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value,
2434 /*is_parallel=*/std::true_type, _IsVector __is_vector)
2435 {
2436 return __internal::__except_handler([&__exec, __first, __last, &__value, __is_vector]() {
2437 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
2438 [&__value, __is_vector](_ForwardIterator __begin, _ForwardIterator __end) {
2439 __internal::__brick_fill(__begin, __end, __value, __is_vector);
2440 });
2441 return __last;
2442 });
2443 }
2444
2445 template <class _OutputIterator, class _Size, class _Tp>
2446 _OutputIterator
__brick_fill_n(_OutputIterator __first,_Size __count,const _Tp & __value,std::true_type)2447 __brick_fill_n(_OutputIterator __first, _Size __count, const _Tp& __value, /* __is_vector = */ std::true_type) noexcept
2448 {
2449 return __unseq_backend::__simd_fill_n(__first, __count, __value);
2450 }
2451
2452 template <class _OutputIterator, class _Size, class _Tp>
2453 _OutputIterator
__brick_fill_n(_OutputIterator __first,_Size __count,const _Tp & __value,std::false_type)2454 __brick_fill_n(_OutputIterator __first, _Size __count, const _Tp& __value, /* __is_vector = */ std::false_type) noexcept
2455 {
2456 return std::fill_n(__first, __count, __value);
2457 }
2458
2459 template <class _ExecutionPolicy, class _OutputIterator, class _Size, class _Tp, class _IsVector>
2460 _OutputIterator
__pattern_fill_n(_ExecutionPolicy &&,_OutputIterator __first,_Size __count,const _Tp & __value,std::false_type,_IsVector __is_vector)2461 __pattern_fill_n(_ExecutionPolicy&&, _OutputIterator __first, _Size __count, const _Tp& __value,
2462 /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept
2463 {
2464 return __internal::__brick_fill_n(__first, __count, __value, __is_vector);
2465 }
2466
2467 template <class _ExecutionPolicy, class _OutputIterator, class _Size, class _Tp, class _IsVector>
2468 _OutputIterator
__pattern_fill_n(_ExecutionPolicy && __exec,_OutputIterator __first,_Size __count,const _Tp & __value,std::true_type,_IsVector __is_vector)2469 __pattern_fill_n(_ExecutionPolicy&& __exec, _OutputIterator __first, _Size __count, const _Tp& __value,
2470 /*is_parallel=*/std::true_type, _IsVector __is_vector)
2471 {
2472 return __internal::__pattern_fill(std::forward<_ExecutionPolicy>(__exec), __first, __first + __count, __value,
2473 std::true_type(), __is_vector);
2474 }
2475
2476 //------------------------------------------------------------------------
2477 // generate, generate_n
2478 //------------------------------------------------------------------------
2479 template <class _RandomAccessIterator, class _Generator>
2480 void
__brick_generate(_RandomAccessIterator __first,_RandomAccessIterator __last,_Generator __g,std::true_type)2481 __brick_generate(_RandomAccessIterator __first, _RandomAccessIterator __last, _Generator __g,
2482 /* is_vector = */ std::true_type) noexcept
2483 {
2484 __unseq_backend::__simd_generate_n(__first, __last - __first, __g);
2485 }
2486
2487 template <class _ForwardIterator, class _Generator>
2488 void
__brick_generate(_ForwardIterator __first,_ForwardIterator __last,_Generator __g,std::false_type)2489 __brick_generate(_ForwardIterator __first, _ForwardIterator __last, _Generator __g,
2490 /* is_vector = */ std::false_type) noexcept
2491 {
2492 std::generate(__first, __last, __g);
2493 }
2494
2495 template <class _ExecutionPolicy, class _ForwardIterator, class _Generator, class _IsVector>
2496 void
__pattern_generate(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Generator __g,std::false_type,_IsVector __is_vector)2497 __pattern_generate(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Generator __g,
2498 /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept
2499 {
2500 __internal::__brick_generate(__first, __last, __g, __is_vector);
2501 }
2502
2503 template <class _ExecutionPolicy, class _ForwardIterator, class _Generator, class _IsVector>
2504 _ForwardIterator
__pattern_generate(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Generator __g,std::true_type,_IsVector __is_vector)2505 __pattern_generate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Generator __g,
2506 /*is_parallel=*/std::true_type, _IsVector __is_vector)
2507 {
2508 return __internal::__except_handler([&]() {
2509 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
2510 [__g, __is_vector](_ForwardIterator __begin, _ForwardIterator __end) {
2511 __internal::__brick_generate(__begin, __end, __g, __is_vector);
2512 });
2513 return __last;
2514 });
2515 }
2516
2517 template <class OutputIterator, class Size, class _Generator>
2518 OutputIterator
__brick_generate_n(OutputIterator __first,Size __count,_Generator __g,std::true_type)2519 __brick_generate_n(OutputIterator __first, Size __count, _Generator __g, /* is_vector = */ std::true_type) noexcept
2520 {
2521 return __unseq_backend::__simd_generate_n(__first, __count, __g);
2522 }
2523
2524 template <class OutputIterator, class Size, class _Generator>
2525 OutputIterator
__brick_generate_n(OutputIterator __first,Size __count,_Generator __g,std::false_type)2526 __brick_generate_n(OutputIterator __first, Size __count, _Generator __g, /* is_vector = */ std::false_type) noexcept
2527 {
2528 return std::generate_n(__first, __count, __g);
2529 }
2530
2531 template <class _ExecutionPolicy, class _OutputIterator, class _Size, class _Generator, class _IsVector>
2532 _OutputIterator
__pattern_generate_n(_ExecutionPolicy &&,_OutputIterator __first,_Size __count,_Generator __g,std::false_type,_IsVector __is_vector)2533 __pattern_generate_n(_ExecutionPolicy&&, _OutputIterator __first, _Size __count, _Generator __g,
2534 /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept
2535 {
2536 return __internal::__brick_generate_n(__first, __count, __g, __is_vector);
2537 }
2538
2539 template <class _ExecutionPolicy, class _OutputIterator, class _Size, class _Generator, class _IsVector>
2540 _OutputIterator
__pattern_generate_n(_ExecutionPolicy && __exec,_OutputIterator __first,_Size __count,_Generator __g,std::true_type,_IsVector __is_vector)2541 __pattern_generate_n(_ExecutionPolicy&& __exec, _OutputIterator __first, _Size __count, _Generator __g,
2542 /*is_parallel=*/std::true_type, _IsVector __is_vector)
2543 {
2544 static_assert(__is_random_access_iterator<_OutputIterator>::value,
2545 "Pattern-brick error. Should be a random access iterator.");
2546 return __internal::__pattern_generate(std::forward<_ExecutionPolicy>(__exec), __first, __first + __count, __g,
2547 std::true_type(), __is_vector);
2548 }
2549
2550 //------------------------------------------------------------------------
2551 // remove
2552 //------------------------------------------------------------------------
2553
2554 template <class _ForwardIterator, class _UnaryPredicate>
2555 _ForwardIterator
__brick_remove_if(_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,std::false_type)2556 __brick_remove_if(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
2557 /* __is_vector = */ std::false_type) noexcept
2558 {
2559 return std::remove_if(__first, __last, __pred);
2560 }
2561
2562 template <class _RandomAccessIterator, class _UnaryPredicate>
2563 _RandomAccessIterator
__brick_remove_if(_RandomAccessIterator __first,_RandomAccessIterator __last,_UnaryPredicate __pred,std::true_type)2564 __brick_remove_if(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryPredicate __pred,
2565 /* __is_vector = */ std::true_type) noexcept
2566 {
2567 #if _PSTL_MONOTONIC_PRESENT
2568 return __unseq_backend::__simd_remove_if(__first, __last - __first, __pred);
2569 #else
2570 return std::remove_if(__first, __last, __pred);
2571 #endif
2572 }
2573
2574 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
2575 _ForwardIterator
__pattern_remove_if(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)2576 __pattern_remove_if(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
2577 _IsVector __is_vector, /*is_parallel*/ std::false_type) noexcept
2578 {
2579 return __internal::__brick_remove_if(__first, __last, __pred, __is_vector);
2580 }
2581
2582 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
2583 _ForwardIterator
__pattern_remove_if(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)2584 __pattern_remove_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
2585 _UnaryPredicate __pred, _IsVector __is_vector, /*is_parallel*/ std::true_type) noexcept
2586 {
2587 typedef typename std::iterator_traits<_ForwardIterator>::reference _ReferenceType;
2588
2589 if (__first == __last || __first + 1 == __last)
2590 {
2591 // Trivial sequence - use serial algorithm
2592 return __internal::__brick_remove_if(__first, __last, __pred, __is_vector);
2593 }
2594
2595 return __internal::__remove_elements(
2596 std::forward<_ExecutionPolicy>(__exec), __first, __last,
2597 [&__pred, __is_vector](bool* __b, bool* __e, _ForwardIterator __it) {
2598 __internal::__brick_walk2(__b, __e, __it, [&__pred](bool& __x, _ReferenceType __y) { __x = !__pred(__y); },
2599 __is_vector);
2600 },
2601 __is_vector);
2602 }
2603
2604 //------------------------------------------------------------------------
2605 // merge
2606 //------------------------------------------------------------------------
2607
2608 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
2609 _OutputIterator
__brick_merge(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __d_first,_Compare __comp,std::false_type)2610 __brick_merge(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
2611 _ForwardIterator2 __last2, _OutputIterator __d_first, _Compare __comp,
2612 /* __is_vector = */ std::false_type) noexcept
2613 {
2614 return std::merge(__first1, __last1, __first2, __last2, __d_first, __comp);
2615 }
2616
2617 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
2618 _OutputIterator
__brick_merge(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __d_first,_Compare __comp,std::true_type)2619 __brick_merge(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
2620 _ForwardIterator2 __last2, _OutputIterator __d_first, _Compare __comp,
2621 /* __is_vector = */ std::true_type) noexcept
2622 {
2623 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
2624 return std::merge(__first1, __last1, __first2, __last2, __d_first, __comp);
2625 }
2626
2627 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
2628 class _Compare, class _IsVector>
2629 _OutputIterator
__pattern_merge(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __d_first,_Compare __comp,_IsVector __is_vector,std::false_type)2630 __pattern_merge(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
2631 _ForwardIterator2 __last2, _OutputIterator __d_first, _Compare __comp, _IsVector __is_vector,
2632 /* is_parallel = */ std::false_type) noexcept
2633 {
2634 return __internal::__brick_merge(__first1, __last1, __first2, __last2, __d_first, __comp, __is_vector);
2635 }
2636
2637 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator,
2638 class _Compare, class _IsVector>
2639 _OutputIterator
__pattern_merge(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator2 __last2,_OutputIterator __d_first,_Compare __comp,_IsVector __is_vector,std::true_type)2640 __pattern_merge(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
2641 _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2, _OutputIterator __d_first,
2642 _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::true_type)
2643 {
2644 __par_backend::__parallel_merge(
2645 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __d_first, __comp,
2646 [__is_vector](_RandomAccessIterator1 __f1, _RandomAccessIterator1 __l1, _RandomAccessIterator2 __f2,
2647 _RandomAccessIterator2 __l2, _OutputIterator __f3, _Compare __comp) {
2648 return __internal::__brick_merge(__f1, __l1, __f2, __l2, __f3, __comp, __is_vector);
2649 });
2650 return __d_first + (__last1 - __first1) + (__last2 - __first2);
2651 }
2652
2653 //------------------------------------------------------------------------
2654 // inplace_merge
2655 //------------------------------------------------------------------------
2656 template <class _BidirectionalIterator, class _Compare>
2657 void
__brick_inplace_merge(_BidirectionalIterator __first,_BidirectionalIterator __middle,_BidirectionalIterator __last,_Compare __comp,std::false_type)2658 __brick_inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last,
2659 _Compare __comp, /* __is_vector = */ std::false_type) noexcept
2660 {
2661 std::inplace_merge(__first, __middle, __last, __comp);
2662 }
2663
2664 template <class _BidirectionalIterator, class _Compare>
2665 void
__brick_inplace_merge(_BidirectionalIterator __first,_BidirectionalIterator __middle,_BidirectionalIterator __last,_Compare __comp,std::true_type)2666 __brick_inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last,
2667 _Compare __comp, /* __is_vector = */ std::true_type) noexcept
2668 {
2669 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial")
2670 std::inplace_merge(__first, __middle, __last, __comp);
2671 }
2672
2673 template <class _ExecutionPolicy, class _BidirectionalIterator, class _Compare, class _IsVector>
2674 void
__pattern_inplace_merge(_ExecutionPolicy &&,_BidirectionalIterator __first,_BidirectionalIterator __middle,_BidirectionalIterator __last,_Compare __comp,_IsVector __is_vector,std::false_type)2675 __pattern_inplace_merge(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __middle,
2676 _BidirectionalIterator __last, _Compare __comp, _IsVector __is_vector,
2677 /* is_parallel = */ std::false_type) noexcept
2678 {
2679 __internal::__brick_inplace_merge(__first, __middle, __last, __comp, __is_vector);
2680 }
2681
2682 template <class _ExecutionPolicy, class _BidirectionalIterator, class _Compare, class _IsVector>
2683 void
__pattern_inplace_merge(_ExecutionPolicy && __exec,_BidirectionalIterator __first,_BidirectionalIterator __middle,_BidirectionalIterator __last,_Compare __comp,_IsVector __is_vector,std::true_type)2684 __pattern_inplace_merge(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __middle,
2685 _BidirectionalIterator __last, _Compare __comp, _IsVector __is_vector,
2686 /*is_parallel=*/std::true_type)
2687 {
2688 if (__first == __last || __first == __middle || __middle == __last)
2689 {
2690 return;
2691 }
2692 typedef typename std::iterator_traits<_BidirectionalIterator>::value_type _Tp;
2693 auto __n = __last - __first;
2694 __par_backend::__buffer<_Tp> __buf(__n);
2695 _Tp* __r = __buf.get();
2696 __internal::__except_handler([&]() {
2697 auto __move_values = [](_BidirectionalIterator __x, _Tp* __z) {
2698 __internal::__invoke_if_else(std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); },
2699 [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); });
2700 };
2701
2702 auto __move_sequences = [](_BidirectionalIterator __first1, _BidirectionalIterator __last1, _Tp* __first2) {
2703 return __internal::__brick_uninitialized_move(__first1, __last1, __first2, _IsVector());
2704 };
2705
2706 __par_backend::__parallel_merge(
2707 std::forward<_ExecutionPolicy>(__exec), __first, __middle, __middle, __last, __r, __comp,
2708 [__n, __move_values, __move_sequences](_BidirectionalIterator __f1, _BidirectionalIterator __l1,
2709 _BidirectionalIterator __f2, _BidirectionalIterator __l2, _Tp* __f3,
2710 _Compare __comp) {
2711 (__utils::__serial_move_merge(__n))(__f1, __l1, __f2, __l2, __f3, __comp, __move_values, __move_values,
2712 __move_sequences, __move_sequences);
2713 return __f3 + (__l1 - __f1) + (__l2 - __f2);
2714 });
2715 __par_backend::__parallel_for(
2716 std::forward<_ExecutionPolicy>(__exec), __r, __r + __n, [__r, __first, __is_vector](_Tp* __i, _Tp* __j) {
2717 __brick_move_destroy()(__i, __j, __first + (__i - __r), __is_vector);
2718 });
2719 });
2720 }
2721
2722 //------------------------------------------------------------------------
2723 // includes
2724 //------------------------------------------------------------------------
2725
2726 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare, class _IsVector>
2727 bool
__pattern_includes(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,_IsVector,std::false_type)2728 __pattern_includes(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
2729 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp, _IsVector,
2730 /*is_parallel=*/std::false_type) noexcept
2731 {
2732 return std::includes(__first1, __last1, __first2, __last2, __comp);
2733 }
2734
2735 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare, class _IsVector>
2736 bool
__pattern_includes(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,_IsVector,std::true_type)2737 __pattern_includes(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
2738 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp, _IsVector,
2739 /*is_parallel=*/std::true_type)
2740 {
2741 if (__first2 >= __last2)
2742 return true;
2743
2744 if (__first1 >= __last1 || __comp(*__first2, *__first1) || __comp(*(__last1 - 1), *(__last2 - 1)))
2745 return false;
2746
2747 __first1 = std::lower_bound(__first1, __last1, *__first2, __comp);
2748 if (__first1 == __last1)
2749 return false;
2750
2751 if (__last2 - __first2 == 1)
2752 return !__comp(*__first1, *__first2) && !__comp(*__first2, *__first1);
2753
2754 return __internal::__except_handler([&]() {
2755 return !__internal::__parallel_or(
2756 std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
2757 [__first1, __last1, __first2, __last2, &__comp](_ForwardIterator2 __i, _ForwardIterator2 __j) {
2758 _PSTL_ASSERT(__j > __i);
2759 //assert(__j - __i > 1);
2760
2761 //1. moving boundaries to "consume" subsequence of equal elements
2762 auto __is_equal = [&__comp](_ForwardIterator2 __a, _ForwardIterator2 __b) -> bool {
2763 return !__comp(*__a, *__b) && !__comp(*__b, *__a);
2764 };
2765
2766 //1.1 left bound, case "aaa[aaaxyz...]" - searching "x"
2767 if (__i > __first2 && __is_equal(__i, __i - 1))
2768 {
2769 //whole subrange continues to content equal elements - return "no op"
2770 if (__is_equal(__i, __j - 1))
2771 return false;
2772
2773 __i = std::upper_bound(__i, __last2, *__i, __comp);
2774 }
2775
2776 //1.2 right bound, case "[...aaa]aaaxyz" - searching "x"
2777 if (__j < __last2 && __is_equal(__j - 1, __j))
2778 __j = std::upper_bound(__j, __last2, *__j, __comp);
2779
2780 //2. testing is __a subsequence of the second range included into the first range
2781 auto __b = std::lower_bound(__first1, __last1, *__i, __comp);
2782
2783 _PSTL_ASSERT(!__comp(*(__last1 - 1), *__b));
2784 _PSTL_ASSERT(!__comp(*(__j - 1), *__i));
2785 return !std::includes(__b, __last1, __i, __j, __comp);
2786 });
2787 });
2788 }
2789
2790 constexpr auto __set_algo_cut_off = 1000;
2791
2792 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
2793 class _Compare, class _IsVector, class _SizeFunction, class _SetOP>
2794 _OutputIterator
__parallel_set_op(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_SizeFunction __size_func,_SetOP __set_op,_IsVector __is_vector)2795 __parallel_set_op(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
2796 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
2797 _SizeFunction __size_func, _SetOP __set_op, _IsVector __is_vector)
2798 {
2799 typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
2800 typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
2801
2802 struct _SetRange
2803 {
2804 _DifferenceType __pos, __len, __buf_pos;
2805 bool
2806 empty() const
2807 {
2808 return __len == 0;
2809 }
2810 };
2811
2812 const _DifferenceType __n1 = __last1 - __first1;
2813 const _DifferenceType __n2 = __last2 - __first2;
2814
2815 __par_backend::__buffer<_Tp> __buf(__size_func(__n1, __n2));
2816
2817 return __internal::__except_handler([&__exec, __n1, __first1, __last1, __first2, __last2, __result, __is_vector,
2818 __comp, __size_func, __set_op, &__buf]() {
2819 auto __buffer = __buf.get();
2820 _DifferenceType __m{};
2821 auto __scan = [=](_DifferenceType, _DifferenceType, const _SetRange& __s) { // Scan
2822 if (!__s.empty())
2823 __brick_move_destroy()(__buffer + __s.__buf_pos,
2824 __buffer + (__s.__buf_pos + __s.__len), __result + __s.__pos,
2825 __is_vector);
2826 };
2827 __par_backend::__parallel_strict_scan(
2828 std::forward<_ExecutionPolicy>(__exec), __n1, _SetRange{0, 0, 0}, //-1, 0},
2829 [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
2830 //[__b; __e) - a subrange of the first sequence, to reduce
2831 _ForwardIterator1 __b = __first1 + __i, __e = __first1 + (__i + __len);
2832
2833 //try searching for the first element which not equal to *__b
2834 if (__b != __first1)
2835 __b = std::upper_bound(__b, __last1, *__b, __comp);
2836
2837 //try searching for the first element which not equal to *__e
2838 if (__e != __last1)
2839 __e = std::upper_bound(__e, __last1, *__e, __comp);
2840
2841 //check is [__b; __e) empty
2842 if (__e - __b < 1)
2843 {
2844 _ForwardIterator2 __bb = __last2;
2845 if (__b != __last1)
2846 __bb = std::lower_bound(__first2, __last2, *__b, __comp);
2847
2848 const _DifferenceType __buf_pos = __size_func((__b - __first1), (__bb - __first2));
2849 return _SetRange{0, 0, __buf_pos};
2850 }
2851
2852 //try searching for "corresponding" subrange [__bb; __ee) in the second sequence
2853 _ForwardIterator2 __bb = __first2;
2854 if (__b != __first1)
2855 __bb = std::lower_bound(__first2, __last2, *__b, __comp);
2856
2857 _ForwardIterator2 __ee = __last2;
2858 if (__e != __last1)
2859 __ee = std::lower_bound(__bb, __last2, *__e, __comp);
2860
2861 const _DifferenceType __buf_pos = __size_func((__b - __first1), (__bb - __first2));
2862 auto __buffer_b = __buffer + __buf_pos;
2863 auto __res = __set_op(__b, __e, __bb, __ee, __buffer_b, __comp);
2864
2865 return _SetRange{0, __res - __buffer_b, __buf_pos};
2866 },
2867 [](const _SetRange& __a, const _SetRange& __b) { // Combine
2868 if (__b.__buf_pos > __a.__buf_pos || ((__b.__buf_pos == __a.__buf_pos) && !__b.empty()))
2869 return _SetRange{__a.__pos + __a.__len + __b.__pos, __b.__len, __b.__buf_pos};
2870 return _SetRange{__b.__pos + __b.__len + __a.__pos, __a.__len, __a.__buf_pos};
2871 },
2872 __scan, // Scan
2873 [&__m, &__scan](const _SetRange& __total) { // Apex
2874 //final scan
2875 __scan(0, 0, __total);
2876 __m = __total.__pos + __total.__len;
2877 });
2878 return __result + __m;
2879 });
2880 }
2881
2882 //a shared parallel pattern for '__pattern_set_union' and '__pattern_set_symmetric_difference'
2883 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
2884 class _Compare, class _SetUnionOp, class _IsVector>
2885 _OutputIterator
__parallel_set_union_op(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_SetUnionOp __set_union_op,_IsVector __is_vector)2886 __parallel_set_union_op(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
2887 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
2888 _Compare __comp, _SetUnionOp __set_union_op, _IsVector __is_vector)
2889 {
2890 typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
2891
2892 const auto __n1 = __last1 - __first1;
2893 const auto __n2 = __last2 - __first2;
2894
2895 auto __copy_range1 = [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) {
2896 return __internal::__brick_copy(__begin, __end, __res, __is_vector);
2897 };
2898 auto __copy_range2 = [__is_vector](_ForwardIterator2 __begin, _ForwardIterator2 __end, _OutputIterator __res) {
2899 return __internal::__brick_copy(__begin, __end, __res, __is_vector);
2900 };
2901
2902 // {1} {}: parallel copying just first sequence
2903 if (__n2 == 0)
2904 return __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
2905 __copy_range1, std::true_type());
2906
2907 // {} {2}: parallel copying justmake second sequence
2908 if (__n1 == 0)
2909 return __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result,
2910 __copy_range2, std::true_type());
2911
2912 // testing whether the sequences are intersected
2913 _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
2914
2915 if (__left_bound_seq_1 == __last1)
2916 {
2917 //{1} < {2}: seq2 is wholly greater than seq1, so, do parallel copying seq1 and seq2
2918 __par_backend::__parallel_invoke(
2919 std::forward<_ExecutionPolicy>(__exec),
2920 [=] {
2921 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
2922 __copy_range1, std::true_type());
2923 },
2924 [=] {
2925 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
2926 __result + __n1, __copy_range2, std::true_type());
2927 });
2928 return __result + __n1 + __n2;
2929 }
2930
2931 // testing whether the sequences are intersected
2932 _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
2933
2934 if (__left_bound_seq_2 == __last2)
2935 {
2936 //{2} < {1}: seq2 is wholly greater than seq1, so, do parallel copying seq1 and seq2
2937 __par_backend::__parallel_invoke(
2938 std::forward<_ExecutionPolicy>(__exec),
2939 [=] {
2940 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result,
2941 __copy_range2, std::true_type());
2942 },
2943 [=] {
2944 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
2945 __result + __n2, __copy_range1, std::true_type());
2946 });
2947 return __result + __n1 + __n2;
2948 }
2949
2950 const auto __m1 = __left_bound_seq_1 - __first1;
2951 if (__m1 > __set_algo_cut_off)
2952 {
2953 auto __res_or = __result;
2954 __result += __m1; //we know proper offset due to [first1; left_bound_seq_1) < [first2; last2)
2955 __par_backend::__parallel_invoke(
2956 std::forward<_ExecutionPolicy>(__exec),
2957 //do parallel copying of [first1; left_bound_seq_1)
2958 [=] {
2959 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __left_bound_seq_1,
2960 __res_or, __copy_range1, std::true_type());
2961 },
2962 [=, &__result] {
2963 __result = __internal::__parallel_set_op(
2964 std::forward<_ExecutionPolicy>(__exec), __left_bound_seq_1, __last1, __first2, __last2, __result,
2965 __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; }, __set_union_op,
2966 __is_vector);
2967 });
2968 return __result;
2969 }
2970
2971 const auto __m2 = __left_bound_seq_2 - __first2;
2972 _PSTL_ASSERT(__m1 == 0 || __m2 == 0);
2973 if (__m2 > __set_algo_cut_off)
2974 {
2975 auto __res_or = __result;
2976 __result += __m2; //we know proper offset due to [first2; left_bound_seq_2) < [first1; last1)
2977 __par_backend::__parallel_invoke(
2978 std::forward<_ExecutionPolicy>(__exec),
2979 //do parallel copying of [first2; left_bound_seq_2)
2980 [=] {
2981 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __left_bound_seq_2,
2982 __res_or, __copy_range2, std::true_type());
2983 },
2984 [=, &__result] {
2985 __result = __internal::__parallel_set_op(
2986 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __left_bound_seq_2, __last2, __result,
2987 __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; }, __set_union_op,
2988 __is_vector);
2989 });
2990 return __result;
2991 }
2992
2993 return __internal::__parallel_set_op(
2994 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
2995 [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; }, __set_union_op, __is_vector);
2996 }
2997
2998 //------------------------------------------------------------------------
2999 // set_union
3000 //------------------------------------------------------------------------
3001
3002 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3003 _OutputIterator
__brick_set_union(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::false_type)3004 __brick_set_union(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3005 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3006 /*__is_vector=*/std::false_type) noexcept
3007 {
3008 return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
3009 }
3010
3011 template <typename _IsVector>
3012 struct __BrickCopyConstruct
3013 {
3014 template <typename _ForwardIterator, typename _OutputIterator>
3015 _OutputIterator
operator__BrickCopyConstruct3016 operator()(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result)
3017 {
3018 return __brick_uninitialized_copy(__first, __last, __result, _IsVector());
3019 }
3020 };
3021
3022 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3023 _OutputIterator
__brick_set_union(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::true_type)3024 __brick_set_union(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3025 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3026 /*__is_vector=*/std::true_type) noexcept
3027 {
3028 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
3029 return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
3030 }
3031
3032 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3033 class _Compare, class _IsVector>
3034 _OutputIterator
__pattern_set_union(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::false_type)3035 __pattern_set_union(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3036 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3037 _IsVector __is_vector,
3038 /*is_parallel=*/std::false_type) noexcept
3039 {
3040 return __internal::__brick_set_union(__first1, __last1, __first2, __last2, __result, __comp, __is_vector);
3041 }
3042
3043 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3044 class _Compare, class _IsVector>
3045 _OutputIterator
__pattern_set_union(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::true_type)3046 __pattern_set_union(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3047 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3048 _IsVector __is_vector, /*__is_parallel=*/std::true_type)
3049 {
3050
3051 const auto __n1 = __last1 - __first1;
3052 const auto __n2 = __last2 - __first2;
3053
3054 // use serial algorithm
3055 if (__n1 + __n2 <= __set_algo_cut_off)
3056 return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
3057
3058 typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
3059 return __parallel_set_union_op(
3060 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
3061 [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2,
3062 _Tp* __result, _Compare __comp) {
3063 return __pstl::__utils::__set_union_construct(__first1, __last1, __first2, __last2, __result, __comp,
3064 __BrickCopyConstruct<_IsVector>());
3065 },
3066 __is_vector);
3067 }
3068
3069 //------------------------------------------------------------------------
3070 // set_intersection
3071 //------------------------------------------------------------------------
3072
3073 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3074 _OutputIterator
__brick_set_intersection(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::false_type)3075 __brick_set_intersection(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3076 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3077 /*__is_vector=*/std::false_type) noexcept
3078 {
3079 return std::set_intersection(__first1, __last1, __first2, __last2, __result, __comp);
3080 }
3081
3082 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3083 _OutputIterator
__brick_set_intersection(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::true_type)3084 __brick_set_intersection(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3085 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3086 /*__is_vector=*/std::true_type) noexcept
3087 {
3088 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
3089 return std::set_intersection(__first1, __last1, __first2, __last2, __result, __comp);
3090 }
3091
3092 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3093 class _Compare, class _IsVector>
3094 _OutputIterator
__pattern_set_intersection(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::false_type)3095 __pattern_set_intersection(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3096 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3097 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
3098 {
3099 return __internal::__brick_set_intersection(__first1, __last1, __first2, __last2, __result, __comp, __is_vector);
3100 }
3101
3102 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3103 class _Compare, class _IsVector>
3104 _OutputIterator
__pattern_set_intersection(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::true_type)3105 __pattern_set_intersection(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3106 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3107 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::true_type)
3108 {
3109 typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
3110 typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
3111
3112 const auto __n1 = __last1 - __first1;
3113 const auto __n2 = __last2 - __first2;
3114
3115 // intersection is empty
3116 if (__n1 == 0 || __n2 == 0)
3117 return __result;
3118
3119 // testing whether the sequences are intersected
3120 _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
3121 //{1} < {2}: seq 2 is wholly greater than seq 1, so, the intersection is empty
3122 if (__left_bound_seq_1 == __last1)
3123 return __result;
3124
3125 // testing whether the sequences are intersected
3126 _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
3127 //{2} < {1}: seq 1 is wholly greater than seq 2, so, the intersection is empty
3128 if (__left_bound_seq_2 == __last2)
3129 return __result;
3130
3131 const auto __m1 = __last1 - __left_bound_seq_1 + __n2;
3132 if (__m1 > __set_algo_cut_off)
3133 {
3134 //we know proper offset due to [first1; left_bound_seq_1) < [first2; last2)
3135 return __internal::__parallel_set_op(
3136 std::forward<_ExecutionPolicy>(__exec), __left_bound_seq_1, __last1, __first2, __last2, __result, __comp,
3137 [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); },
3138 [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3139 _ForwardIterator2 __last2, _Tp* __result, _Compare __comp) {
3140 return __pstl::__utils::__set_intersection_construct(__first1, __last1, __first2, __last2, __result,
3141 __comp);
3142 },
3143 __is_vector);
3144 }
3145
3146 const auto __m2 = __last2 - __left_bound_seq_2 + __n1;
3147 if (__m2 > __set_algo_cut_off)
3148 {
3149 //we know proper offset due to [first2; left_bound_seq_2) < [first1; last1)
3150 __result = __internal::__parallel_set_op(
3151 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __left_bound_seq_2, __last2, __result, __comp,
3152 [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); },
3153 [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3154 _ForwardIterator2 __last2, _Tp* __result, _Compare __comp) {
3155 return __pstl::__utils::__set_intersection_construct(__first2, __last2, __first1, __last1, __result,
3156 __comp);
3157 },
3158 __is_vector);
3159 return __result;
3160 }
3161
3162 // [left_bound_seq_1; last1) and [left_bound_seq_2; last2) - use serial algorithm
3163 return std::set_intersection(__left_bound_seq_1, __last1, __left_bound_seq_2, __last2, __result, __comp);
3164 }
3165
3166 //------------------------------------------------------------------------
3167 // set_difference
3168 //------------------------------------------------------------------------
3169
3170 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3171 _OutputIterator
__brick_set_difference(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::false_type)3172 __brick_set_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3173 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3174 /*__is_vector=*/std::false_type) noexcept
3175 {
3176 return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
3177 }
3178
3179 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3180 _OutputIterator
__brick_set_difference(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::true_type)3181 __brick_set_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3182 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3183 /*__is_vector=*/std::true_type) noexcept
3184 {
3185 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
3186 return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
3187 }
3188
3189 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3190 class _Compare, class _IsVector>
3191 _OutputIterator
__pattern_set_difference(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::false_type)3192 __pattern_set_difference(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3193 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3194 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
3195 {
3196 return __internal::__brick_set_difference(__first1, __last1, __first2, __last2, __result, __comp, __is_vector);
3197 }
3198
3199 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3200 class _Compare, class _IsVector>
3201 _OutputIterator
__pattern_set_difference(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::true_type)3202 __pattern_set_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3203 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3204 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::true_type)
3205 {
3206 typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
3207 typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
3208
3209 const auto __n1 = __last1 - __first1;
3210 const auto __n2 = __last2 - __first2;
3211
3212 // {} \ {2}: the difference is empty
3213 if (__n1 == 0)
3214 return __result;
3215
3216 // {1} \ {}: parallel copying just first sequence
3217 if (__n2 == 0)
3218 return __internal::__pattern_walk2_brick(
3219 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
3220 [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) {
3221 return __internal::__brick_copy(__begin, __end, __res, __is_vector);
3222 },
3223 std::true_type());
3224
3225 // testing whether the sequences are intersected
3226 _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
3227 //{1} < {2}: seq 2 is wholly greater than seq 1, so, parallel copying just first sequence
3228 if (__left_bound_seq_1 == __last1)
3229 return __internal::__pattern_walk2_brick(
3230 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
3231 [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) {
3232 return __internal::__brick_copy(__begin, __end, __res, __is_vector);
3233 },
3234 std::true_type());
3235
3236 // testing whether the sequences are intersected
3237 _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
3238 //{2} < {1}: seq 1 is wholly greater than seq 2, so, parallel copying just first sequence
3239 if (__left_bound_seq_2 == __last2)
3240 return __internal::__pattern_walk2_brick(
3241 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
3242 [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) {
3243 return __internal::__brick_copy(__begin, __end, __res, __is_vector);
3244 },
3245 std::true_type());
3246
3247 if (__n1 + __n2 > __set_algo_cut_off)
3248 return __parallel_set_op(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result,
3249 __comp, [](_DifferenceType __n, _DifferenceType) { return __n; },
3250 [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3251 _ForwardIterator2 __last2, _Tp* __result, _Compare __comp) {
3252 return __pstl::__utils::__set_difference_construct(
3253 __first1, __last1, __first2, __last2, __result, __comp,
3254 __BrickCopyConstruct<_IsVector>());
3255 },
3256 __is_vector);
3257
3258 // use serial algorithm
3259 return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
3260 }
3261
3262 //------------------------------------------------------------------------
3263 // set_symmetric_difference
3264 //------------------------------------------------------------------------
3265
3266 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3267 _OutputIterator
__brick_set_symmetric_difference(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::false_type)3268 __brick_set_symmetric_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3269 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3270 /*__is_vector=*/std::false_type) noexcept
3271 {
3272 return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
3273 }
3274
3275 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3276 _OutputIterator
__brick_set_symmetric_difference(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::true_type)3277 __brick_set_symmetric_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3278 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3279 /*__is_vector=*/std::true_type) noexcept
3280 {
3281 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
3282 return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
3283 }
3284
3285 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3286 class _Compare, class _IsVector>
3287 _OutputIterator
__pattern_set_symmetric_difference(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::false_type)3288 __pattern_set_symmetric_difference(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3289 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3290 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
3291 {
3292 return __internal::__brick_set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp,
3293 __is_vector);
3294 }
3295
3296 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3297 class _Compare, class _IsVector>
3298 _OutputIterator
__pattern_set_symmetric_difference(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::true_type)3299 __pattern_set_symmetric_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3300 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3301 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::true_type)
3302 {
3303
3304 const auto __n1 = __last1 - __first1;
3305 const auto __n2 = __last2 - __first2;
3306
3307 // use serial algorithm
3308 if (__n1 + __n2 <= __set_algo_cut_off)
3309 return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
3310
3311 typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
3312 return __internal::__parallel_set_union_op(
3313 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
3314 [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2,
3315 _Tp* __result, _Compare __comp) {
3316 return __pstl::__utils::__set_symmetric_difference_construct(__first1, __last1, __first2, __last2, __result,
3317 __comp, __BrickCopyConstruct<_IsVector>());
3318 },
3319 __is_vector);
3320 }
3321
3322 //------------------------------------------------------------------------
3323 // is_heap_until
3324 //------------------------------------------------------------------------
3325
3326 template <class _RandomAccessIterator, class _Compare>
3327 _RandomAccessIterator
__brick_is_heap_until(_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,std::false_type)3328 __brick_is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
3329 /* __is_vector = */ std::false_type) noexcept
3330 {
3331 return std::is_heap_until(__first, __last, __comp);
3332 }
3333
3334 template <class _RandomAccessIterator, class _Compare>
3335 _RandomAccessIterator
__brick_is_heap_until(_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,std::true_type)3336 __brick_is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
3337 /* __is_vector = */ std::true_type) noexcept
3338 {
3339 if (__last - __first < 2)
3340 return __last;
3341 typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
3342 return __unseq_backend::__simd_first(
3343 __first, _SizeType(0), __last - __first,
3344 [&__comp](_RandomAccessIterator __it, _SizeType __i) { return __comp(__it[(__i - 1) / 2], __it[__i]); });
3345 }
3346
3347 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
3348 _RandomAccessIterator
__pattern_is_heap_until(_ExecutionPolicy &&,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector __is_vector,std::false_type)3349 __pattern_is_heap_until(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last,
3350 _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept
3351 {
3352 return __internal::__brick_is_heap_until(__first, __last, __comp, __is_vector);
3353 }
3354
3355 template <class _RandomAccessIterator, class _DifferenceType, class _Compare>
3356 _RandomAccessIterator
__is_heap_until_local(_RandomAccessIterator __first,_DifferenceType __begin,_DifferenceType __end,_Compare __comp,std::false_type)3357 __is_heap_until_local(_RandomAccessIterator __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp,
3358 /* __is_vector = */ std::false_type) noexcept
3359 {
3360 _DifferenceType __i = __begin;
3361 for (; __i < __end; ++__i)
3362 {
3363 if (__comp(__first[(__i - 1) / 2], __first[__i]))
3364 {
3365 break;
3366 }
3367 }
3368 return __first + __i;
3369 }
3370
3371 template <class _RandomAccessIterator, class _DifferenceType, class _Compare>
3372 _RandomAccessIterator
__is_heap_until_local(_RandomAccessIterator __first,_DifferenceType __begin,_DifferenceType __end,_Compare __comp,std::true_type)3373 __is_heap_until_local(_RandomAccessIterator __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp,
3374 /* __is_vector = */ std::true_type) noexcept
3375 {
3376 return __unseq_backend::__simd_first(
3377 __first, __begin, __end,
3378 [&__comp](_RandomAccessIterator __it, _DifferenceType __i) { return __comp(__it[(__i - 1) / 2], __it[__i]); });
3379 }
3380
3381 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
3382 _RandomAccessIterator
__pattern_is_heap_until(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector __is_vector,std::true_type)3383 __pattern_is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
3384 _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::true_type) noexcept
3385 {
3386 if (__last - __first < 2)
3387 return __last;
3388
3389 return __internal::__except_handler([&]() {
3390 return __parallel_find(
3391 std::forward<_ExecutionPolicy>(__exec), __first, __last,
3392 [__first, __comp, __is_vector](_RandomAccessIterator __i, _RandomAccessIterator __j) {
3393 return __internal::__is_heap_until_local(__first, __i - __first, __j - __first, __comp, __is_vector);
3394 },
3395 std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(), /*is_first=*/true);
3396 });
3397 }
3398
3399 //------------------------------------------------------------------------
3400 // min_element
3401 //------------------------------------------------------------------------
3402
3403 template <typename _ForwardIterator, typename _Compare>
3404 _ForwardIterator
__brick_min_element(_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,std::false_type)3405 __brick_min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3406 /* __is_vector = */ std::false_type) noexcept
3407 {
3408 return std::min_element(__first, __last, __comp);
3409 }
3410
3411 template <typename _ForwardIterator, typename _Compare>
3412 _ForwardIterator
__brick_min_element(_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,std::true_type)3413 __brick_min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3414 /* __is_vector = */ std::true_type) noexcept
3415 {
3416 #if _PSTL_UDR_PRESENT
3417 return __unseq_backend::__simd_min_element(__first, __last - __first, __comp);
3418 #else
3419 return std::min_element(__first, __last, __comp);
3420 #endif
3421 }
3422
3423 template <typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare, typename _IsVector>
3424 _ForwardIterator
__pattern_min_element(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,_IsVector __is_vector,std::false_type)3425 __pattern_min_element(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3426 _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept
3427 {
3428 return __internal::__brick_min_element(__first, __last, __comp, __is_vector);
3429 }
3430
3431 template <typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare, typename _IsVector>
3432 _RandomAccessIterator
__pattern_min_element(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector __is_vector,std::true_type)3433 __pattern_min_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
3434 _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::true_type)
3435 {
3436 if (__first == __last)
3437 return __last;
3438
3439 return __internal::__except_handler([&]() {
3440 return __par_backend::__parallel_reduce(
3441 std::forward<_ExecutionPolicy>(__exec), __first + 1, __last, __first,
3442 [=](_RandomAccessIterator __begin, _RandomAccessIterator __end,
3443 _RandomAccessIterator __init) -> _RandomAccessIterator {
3444 const _RandomAccessIterator subresult =
3445 __internal::__brick_min_element(__begin, __end, __comp, __is_vector);
3446 return __internal::__cmp_iterators_by_values(__init, subresult, __comp);
3447 },
3448 [=](_RandomAccessIterator __it1, _RandomAccessIterator __it2) -> _RandomAccessIterator {
3449 return __internal::__cmp_iterators_by_values(__it1, __it2, __comp);
3450 });
3451 });
3452 }
3453
3454 //------------------------------------------------------------------------
3455 // minmax_element
3456 //------------------------------------------------------------------------
3457
3458 template <typename _ForwardIterator, typename _Compare>
3459 std::pair<_ForwardIterator, _ForwardIterator>
__brick_minmax_element(_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,std::false_type)3460 __brick_minmax_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3461 /* __is_vector = */ std::false_type) noexcept
3462 {
3463 return std::minmax_element(__first, __last, __comp);
3464 }
3465
3466 template <typename _ForwardIterator, typename _Compare>
3467 std::pair<_ForwardIterator, _ForwardIterator>
__brick_minmax_element(_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,std::true_type)3468 __brick_minmax_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3469 /* __is_vector = */ std::true_type) noexcept
3470 {
3471 #if _PSTL_UDR_PRESENT
3472 return __unseq_backend::__simd_minmax_element(__first, __last - __first, __comp);
3473 #else
3474 return std::minmax_element(__first, __last, __comp);
3475 #endif
3476 }
3477
3478 template <typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare, typename _IsVector>
3479 std::pair<_ForwardIterator, _ForwardIterator>
__pattern_minmax_element(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,_IsVector __is_vector,std::false_type)3480 __pattern_minmax_element(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3481 _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept
3482 {
3483 return __internal::__brick_minmax_element(__first, __last, __comp, __is_vector);
3484 }
3485
3486 template <typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare, typename _IsVector>
3487 std::pair<_ForwardIterator, _ForwardIterator>
__pattern_minmax_element(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,_IsVector __is_vector,std::true_type)3488 __pattern_minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3489 _IsVector __is_vector, /* is_parallel = */ std::true_type)
3490 {
3491 if (__first == __last)
3492 return std::make_pair(__first, __first);
3493
3494 return __internal::__except_handler([&]() {
3495 typedef std::pair<_ForwardIterator, _ForwardIterator> _Result;
3496
3497 return __par_backend::__parallel_reduce(
3498 std::forward<_ExecutionPolicy>(__exec), __first + 1, __last, std::make_pair(__first, __first),
3499 [=](_ForwardIterator __begin, _ForwardIterator __end, _Result __init) -> _Result {
3500 const _Result __subresult = __internal::__brick_minmax_element(__begin, __end, __comp, __is_vector);
3501 return std::make_pair(
3502 __internal::__cmp_iterators_by_values(__subresult.first, __init.first, __comp),
3503 __internal::__cmp_iterators_by_values(__init.second, __subresult.second, std::not_fn(__comp)));
3504 },
3505 [=](_Result __p1, _Result __p2) -> _Result {
3506 return std::make_pair(
3507 __internal::__cmp_iterators_by_values(__p1.first, __p2.first, __comp),
3508 __internal::__cmp_iterators_by_values(__p2.second, __p1.second, std::not_fn(__comp)));
3509 });
3510 });
3511 }
3512
3513 //------------------------------------------------------------------------
3514 // mismatch
3515 //------------------------------------------------------------------------
3516 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
3517 std::pair<_ForwardIterator1, _ForwardIterator2>
__mismatch_serial(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_BinaryPredicate __pred)3518 __mismatch_serial(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3519 _ForwardIterator2 __last2, _BinaryPredicate __pred)
3520 {
3521 #if _PSTL_CPP14_2RANGE_MISMATCH_EQUAL_PRESENT
3522 return std::mismatch(__first1, __last1, __first2, __last2, __pred);
3523 #else
3524 for (; __first1 != __last1 && __first2 != __last2 && __pred(*__first1, *__first2); ++__first1, ++__first2)
3525 {
3526 }
3527 return std::make_pair(__first1, __first2);
3528 #endif
3529 }
3530
3531 template <class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
3532 std::pair<_ForwardIterator1, _ForwardIterator2>
__brick_mismatch(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Predicate __pred,std::false_type)3533 __brick_mismatch(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3534 _ForwardIterator2 __last2, _Predicate __pred, /* __is_vector = */ std::false_type) noexcept
3535 {
3536 return __mismatch_serial(__first1, __last1, __first2, __last2, __pred);
3537 }
3538
3539 template <class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
3540 std::pair<_ForwardIterator1, _ForwardIterator2>
__brick_mismatch(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Predicate __pred,std::true_type)3541 __brick_mismatch(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3542 _ForwardIterator2 __last2, _Predicate __pred, /* __is_vector = */ std::true_type) noexcept
3543 {
3544 auto __n = std::min(__last1 - __first1, __last2 - __first2);
3545 return __unseq_backend::__simd_first(__first1, __n, __first2, std::not_fn(__pred));
3546 }
3547
3548 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate, class _IsVector>
3549 std::pair<_ForwardIterator1, _ForwardIterator2>
__pattern_mismatch(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Predicate __pred,_IsVector __is_vector,std::false_type)3550 __pattern_mismatch(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3551 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Predicate __pred, _IsVector __is_vector,
3552 /* is_parallel = */ std::false_type) noexcept
3553 {
3554 return __internal::__brick_mismatch(__first1, __last1, __first2, __last2, __pred, __is_vector);
3555 }
3556
3557 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _Predicate,
3558 class _IsVector>
3559 std::pair<_RandomAccessIterator1, _RandomAccessIterator2>
__pattern_mismatch(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator2 __last2,_Predicate __pred,_IsVector __is_vector,std::true_type)3560 __pattern_mismatch(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
3561 _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2, _Predicate __pred,
3562 _IsVector __is_vector, /* is_parallel = */ std::true_type) noexcept
3563 {
3564 return __internal::__except_handler([&]() {
3565 auto __n = std::min(__last1 - __first1, __last2 - __first2);
3566 auto __result = __internal::__parallel_find(
3567 std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
3568 [__first1, __first2, __pred, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
3569 return __internal::__brick_mismatch(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1),
3570 __pred, __is_vector)
3571 .first;
3572 },
3573 std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(), /*is_first=*/true);
3574 return std::make_pair(__result, __first2 + (__result - __first1));
3575 });
3576 }
3577
3578 //------------------------------------------------------------------------
3579 // lexicographical_compare
3580 //------------------------------------------------------------------------
3581
3582 template <class _ForwardIterator1, class _ForwardIterator2, class _Compare>
3583 bool
__brick_lexicographical_compare(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,std::false_type)3584 __brick_lexicographical_compare(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3585 _ForwardIterator2 __last2, _Compare __comp,
3586 /* __is_vector = */ std::false_type) noexcept
3587 {
3588 return std::lexicographical_compare(__first1, __last1, __first2, __last2, __comp);
3589 }
3590
3591 template <class _ForwardIterator1, class _ForwardIterator2, class _Compare>
3592 bool
__brick_lexicographical_compare(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,std::true_type)3593 __brick_lexicographical_compare(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3594 _ForwardIterator2 __last2, _Compare __comp, /* __is_vector = */ std::true_type) noexcept
3595 {
3596 if (__first2 == __last2)
3597 { // if second sequence is empty
3598 return false;
3599 }
3600 else if (__first1 == __last1)
3601 { // if first sequence is empty
3602 return true;
3603 }
3604 else
3605 {
3606 typedef typename std::iterator_traits<_ForwardIterator1>::reference ref_type1;
3607 typedef typename std::iterator_traits<_ForwardIterator2>::reference ref_type2;
3608 --__last1;
3609 --__last2;
3610 auto __n = std::min(__last1 - __first1, __last2 - __first2);
3611 std::pair<_ForwardIterator1, _ForwardIterator2> __result = __unseq_backend::__simd_first(
3612 __first1, __n, __first2, [__comp](const ref_type1 __x, const ref_type2 __y) mutable {
3613 return __comp(__x, __y) || __comp(__y, __x);
3614 });
3615
3616 if (__result.first == __last1 && __result.second != __last2)
3617 { // if first sequence shorter than second
3618 return !__comp(*__result.second, *__result.first);
3619 }
3620 else
3621 { // if second sequence shorter than first or both have the same number of elements
3622 return __comp(*__result.first, *__result.second);
3623 }
3624 }
3625 }
3626
3627 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare, class _IsVector>
3628 bool
__pattern_lexicographical_compare(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,_IsVector __is_vector,std::false_type)3629 __pattern_lexicographical_compare(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3630 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp,
3631 _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept
3632 {
3633 return __internal::__brick_lexicographical_compare(__first1, __last1, __first2, __last2, __comp, __is_vector);
3634 }
3635
3636 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare, class _IsVector>
3637 bool
__pattern_lexicographical_compare(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,_IsVector __is_vector,std::true_type)3638 __pattern_lexicographical_compare(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3639 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp,
3640 _IsVector __is_vector, /* is_parallel = */ std::true_type) noexcept
3641 {
3642 if (__first2 == __last2)
3643 { // if second sequence is empty
3644 return false;
3645 }
3646 else if (__first1 == __last1)
3647 { // if first sequence is empty
3648 return true;
3649 }
3650 else
3651 {
3652 typedef typename std::iterator_traits<_ForwardIterator1>::reference _RefType1;
3653 typedef typename std::iterator_traits<_ForwardIterator2>::reference _RefType2;
3654 --__last1;
3655 --__last2;
3656 auto __n = std::min(__last1 - __first1, __last2 - __first2);
3657 auto __result = __internal::__parallel_find(
3658 std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
3659 [__first1, __first2, &__comp, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
3660 return __internal::__brick_mismatch(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1),
3661 [&__comp](const _RefType1 __x, const _RefType2 __y) {
3662 return !__comp(__x, __y) && !__comp(__y, __x);
3663 },
3664 __is_vector)
3665 .first;
3666 },
3667 std::less<typename std::iterator_traits<_ForwardIterator1>::difference_type>(), /*is_first=*/true);
3668
3669 if (__result == __last1 && __first2 + (__result - __first1) != __last2)
3670 { // if first sequence shorter than second
3671 return !__comp(*(__first2 + (__result - __first1)), *__result);
3672 }
3673 else
3674 { // if second sequence shorter than first or both have the same number of elements
3675 return __comp(*__result, *(__first2 + (__result - __first1)));
3676 }
3677 }
3678 }
3679
3680 } // namespace __internal
3681 } // namespace __pstl
3682
3683 #endif /* _PSTL_ALGORITHM_IMPL_H */
3684