1 /* Copyright (C) 2015-2018 Free Software Foundation, Inc. 2 Contributed by Jakub Jelinek <jakub@redhat.com>. 3 4 This file is part of the GNU Offloading and Multi Processing Library 5 (libgomp). 6 7 Libgomp is free software; you can redistribute it and/or modify it 8 under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 3, or (at your option) 10 any later version. 11 12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY 13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 15 more details. 16 17 Under Section 7 of GPL version 3, you are granted additional 18 permissions described in the GCC Runtime Library Exception, version 19 3.1, as published by the Free Software Foundation. 20 21 You should have received a copy of the GNU General Public License and 22 a copy of the GCC Runtime Library Exception along with this program; 23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 24 <http://www.gnu.org/licenses/>. */ 25 26 /* This file handles the taskloop construct. It is included twice, once 27 for the long and once for unsigned long long variant. */ 28 29 /* Called when encountering an explicit task directive. If IF_CLAUSE is 30 false, then we must not delay in executing the task. If UNTIED is true, 31 then the task may be executed by any member of the team. */ 32 33 void 34 GOMP_taskloop (void (*fn) (void *), void *data, void (*cpyfn) (void *, void *), 35 long arg_size, long arg_align, unsigned flags, 36 unsigned long num_tasks, int priority, 37 TYPE start, TYPE end, TYPE step) 38 { 39 struct gomp_thread *thr = gomp_thread (); 40 struct gomp_team *team = thr->ts.team; 41 42 #ifdef HAVE_BROKEN_POSIX_SEMAPHORES 43 /* If pthread_mutex_* is used for omp_*lock*, then each task must be 44 tied to one thread all the time. This means UNTIED tasks must be 45 tied and if CPYFN is non-NULL IF(0) must be forced, as CPYFN 46 might be running on different thread than FN. */ 47 if (cpyfn) 48 flags &= ~GOMP_TASK_FLAG_IF; 49 flags &= ~GOMP_TASK_FLAG_UNTIED; 50 #endif 51 52 /* If parallel or taskgroup has been cancelled, don't start new tasks. */ 53 if (team && gomp_team_barrier_cancelled (&team->barrier)) 54 return; 55 56 #ifdef TYPE_is_long 57 TYPE s = step; 58 if (step > 0) 59 { 60 if (start >= end) 61 return; 62 s--; 63 } 64 else 65 { 66 if (start <= end) 67 return; 68 s++; 69 } 70 UTYPE n = (end - start + s) / step; 71 #else 72 UTYPE n; 73 if (flags & GOMP_TASK_FLAG_UP) 74 { 75 if (start >= end) 76 return; 77 n = (end - start + step - 1) / step; 78 } 79 else 80 { 81 if (start <= end) 82 return; 83 n = (start - end - step - 1) / -step; 84 } 85 #endif 86 87 TYPE task_step = step; 88 unsigned long nfirst = n; 89 if (flags & GOMP_TASK_FLAG_GRAINSIZE) 90 { 91 unsigned long grainsize = num_tasks; 92 #ifdef TYPE_is_long 93 num_tasks = n / grainsize; 94 #else 95 UTYPE ndiv = n / grainsize; 96 num_tasks = ndiv; 97 if (num_tasks != ndiv) 98 num_tasks = ~0UL; 99 #endif 100 if (num_tasks <= 1) 101 { 102 num_tasks = 1; 103 task_step = end - start; 104 } 105 else if (num_tasks >= grainsize 106 #ifndef TYPE_is_long 107 && num_tasks != ~0UL 108 #endif 109 ) 110 { 111 UTYPE mul = num_tasks * grainsize; 112 task_step = (TYPE) grainsize * step; 113 if (mul != n) 114 { 115 task_step += step; 116 nfirst = n - mul - 1; 117 } 118 } 119 else 120 { 121 UTYPE div = n / num_tasks; 122 UTYPE mod = n % num_tasks; 123 task_step = (TYPE) div * step; 124 if (mod) 125 { 126 task_step += step; 127 nfirst = mod - 1; 128 } 129 } 130 } 131 else 132 { 133 if (num_tasks == 0) 134 num_tasks = team ? team->nthreads : 1; 135 if (num_tasks >= n) 136 num_tasks = n; 137 else 138 { 139 UTYPE div = n / num_tasks; 140 UTYPE mod = n % num_tasks; 141 task_step = (TYPE) div * step; 142 if (mod) 143 { 144 task_step += step; 145 nfirst = mod - 1; 146 } 147 } 148 } 149 150 if (flags & GOMP_TASK_FLAG_NOGROUP) 151 { 152 if (thr->task && thr->task->taskgroup && thr->task->taskgroup->cancelled) 153 return; 154 } 155 else 156 ialias_call (GOMP_taskgroup_start) (); 157 158 if (priority > gomp_max_task_priority_var) 159 priority = gomp_max_task_priority_var; 160 161 if ((flags & GOMP_TASK_FLAG_IF) == 0 || team == NULL 162 || (thr->task && thr->task->final_task) 163 || team->task_count + num_tasks > 64 * team->nthreads) 164 { 165 unsigned long i; 166 if (__builtin_expect (cpyfn != NULL, 0)) 167 { 168 struct gomp_task task[num_tasks]; 169 struct gomp_task *parent = thr->task; 170 arg_size = (arg_size + arg_align - 1) & ~(arg_align - 1); 171 char buf[num_tasks * arg_size + arg_align - 1]; 172 char *arg = (char *) (((uintptr_t) buf + arg_align - 1) 173 & ~(uintptr_t) (arg_align - 1)); 174 char *orig_arg = arg; 175 for (i = 0; i < num_tasks; i++) 176 { 177 gomp_init_task (&task[i], parent, gomp_icv (false)); 178 task[i].priority = priority; 179 task[i].kind = GOMP_TASK_UNDEFERRED; 180 task[i].final_task = (thr->task && thr->task->final_task) 181 || (flags & GOMP_TASK_FLAG_FINAL); 182 if (thr->task) 183 { 184 task[i].in_tied_task = thr->task->in_tied_task; 185 task[i].taskgroup = thr->task->taskgroup; 186 } 187 thr->task = &task[i]; 188 cpyfn (arg, data); 189 arg += arg_size; 190 } 191 arg = orig_arg; 192 for (i = 0; i < num_tasks; i++) 193 { 194 thr->task = &task[i]; 195 ((TYPE *)arg)[0] = start; 196 start += task_step; 197 ((TYPE *)arg)[1] = start; 198 if (i == nfirst) 199 task_step -= step; 200 fn (arg); 201 arg += arg_size; 202 if (!priority_queue_empty_p (&task[i].children_queue, 203 MEMMODEL_RELAXED)) 204 { 205 gomp_mutex_lock (&team->task_lock); 206 gomp_clear_parent (&task[i].children_queue); 207 gomp_mutex_unlock (&team->task_lock); 208 } 209 gomp_end_task (); 210 } 211 } 212 else 213 for (i = 0; i < num_tasks; i++) 214 { 215 struct gomp_task task; 216 217 gomp_init_task (&task, thr->task, gomp_icv (false)); 218 task.priority = priority; 219 task.kind = GOMP_TASK_UNDEFERRED; 220 task.final_task = (thr->task && thr->task->final_task) 221 || (flags & GOMP_TASK_FLAG_FINAL); 222 if (thr->task) 223 { 224 task.in_tied_task = thr->task->in_tied_task; 225 task.taskgroup = thr->task->taskgroup; 226 } 227 thr->task = &task; 228 ((TYPE *)data)[0] = start; 229 start += task_step; 230 ((TYPE *)data)[1] = start; 231 if (i == nfirst) 232 task_step -= step; 233 fn (data); 234 if (!priority_queue_empty_p (&task.children_queue, 235 MEMMODEL_RELAXED)) 236 { 237 gomp_mutex_lock (&team->task_lock); 238 gomp_clear_parent (&task.children_queue); 239 gomp_mutex_unlock (&team->task_lock); 240 } 241 gomp_end_task (); 242 } 243 } 244 else 245 { 246 struct gomp_task *tasks[num_tasks]; 247 struct gomp_task *parent = thr->task; 248 struct gomp_taskgroup *taskgroup = parent->taskgroup; 249 char *arg; 250 int do_wake; 251 unsigned long i; 252 253 for (i = 0; i < num_tasks; i++) 254 { 255 struct gomp_task *task 256 = gomp_malloc (sizeof (*task) + arg_size + arg_align - 1); 257 tasks[i] = task; 258 arg = (char *) (((uintptr_t) (task + 1) + arg_align - 1) 259 & ~(uintptr_t) (arg_align - 1)); 260 gomp_init_task (task, parent, gomp_icv (false)); 261 task->priority = priority; 262 task->kind = GOMP_TASK_UNDEFERRED; 263 task->in_tied_task = parent->in_tied_task; 264 task->taskgroup = taskgroup; 265 thr->task = task; 266 if (cpyfn) 267 { 268 cpyfn (arg, data); 269 task->copy_ctors_done = true; 270 } 271 else 272 memcpy (arg, data, arg_size); 273 ((TYPE *)arg)[0] = start; 274 start += task_step; 275 ((TYPE *)arg)[1] = start; 276 if (i == nfirst) 277 task_step -= step; 278 thr->task = parent; 279 task->kind = GOMP_TASK_WAITING; 280 task->fn = fn; 281 task->fn_data = arg; 282 task->final_task = (flags & GOMP_TASK_FLAG_FINAL) >> 1; 283 } 284 gomp_mutex_lock (&team->task_lock); 285 /* If parallel or taskgroup has been cancelled, don't start new 286 tasks. */ 287 if (__builtin_expect ((gomp_team_barrier_cancelled (&team->barrier) 288 || (taskgroup && taskgroup->cancelled)) 289 && cpyfn == NULL, 0)) 290 { 291 gomp_mutex_unlock (&team->task_lock); 292 for (i = 0; i < num_tasks; i++) 293 { 294 gomp_finish_task (tasks[i]); 295 free (tasks[i]); 296 } 297 if ((flags & GOMP_TASK_FLAG_NOGROUP) == 0) 298 ialias_call (GOMP_taskgroup_end) (); 299 return; 300 } 301 if (taskgroup) 302 taskgroup->num_children += num_tasks; 303 for (i = 0; i < num_tasks; i++) 304 { 305 struct gomp_task *task = tasks[i]; 306 priority_queue_insert (PQ_CHILDREN, &parent->children_queue, 307 task, priority, 308 PRIORITY_INSERT_BEGIN, 309 /*last_parent_depends_on=*/false, 310 task->parent_depends_on); 311 if (taskgroup) 312 priority_queue_insert (PQ_TASKGROUP, &taskgroup->taskgroup_queue, 313 task, priority, PRIORITY_INSERT_BEGIN, 314 /*last_parent_depends_on=*/false, 315 task->parent_depends_on); 316 priority_queue_insert (PQ_TEAM, &team->task_queue, task, priority, 317 PRIORITY_INSERT_END, 318 /*last_parent_depends_on=*/false, 319 task->parent_depends_on); 320 ++team->task_count; 321 ++team->task_queued_count; 322 } 323 gomp_team_barrier_set_task_pending (&team->barrier); 324 if (team->task_running_count + !parent->in_tied_task 325 < team->nthreads) 326 { 327 do_wake = team->nthreads - team->task_running_count 328 - !parent->in_tied_task; 329 if ((unsigned long) do_wake > num_tasks) 330 do_wake = num_tasks; 331 } 332 else 333 do_wake = 0; 334 gomp_mutex_unlock (&team->task_lock); 335 if (do_wake) 336 gomp_team_barrier_wake (&team->barrier, do_wake); 337 } 338 if ((flags & GOMP_TASK_FLAG_NOGROUP) == 0) 339 ialias_call (GOMP_taskgroup_end) (); 340 } 341