Lines Matching refs:aq
299 struct goacc_asyncqueue *aq; member
2100 struct goacc_asyncqueue *aq, bool module_locked) in run_kernel() argument
2121 (aq ? aq->id : 0)); in run_kernel()
2157 hsa_queue_t *command_q = (aq ? aq->hsa_queue : kernel->agent->sync_queue); in run_kernel()
2735 execute_queue_entry (struct goacc_asyncqueue *aq, int index) in execute_queue_entry() argument
2737 struct queue_entry *entry = &aq->queue[index]; in execute_queue_entry()
2744 aq->agent->device_id, aq->id, index); in execute_queue_entry()
2747 &entry->u.launch.kla, aq, false); in execute_queue_entry()
2750 aq->agent->device_id, aq->id, index); in execute_queue_entry()
2756 aq->agent->device_id, aq->id, index); in execute_queue_entry()
2760 aq->agent->device_id, aq->id, index); in execute_queue_entry()
2771 aq->agent->device_id, aq->id, index); in execute_queue_entry()
2788 "entry (%d) done\n", aq->agent->device_id, aq->id, index); in execute_queue_entry()
2810 struct goacc_asyncqueue *aq = thread_arg; in drain_queue() local
2814 aq->drain_queue_stop = 2; in drain_queue()
2818 pthread_mutex_lock (&aq->mutex); in drain_queue()
2822 if (aq->drain_queue_stop) in drain_queue()
2825 if (aq->queue_n > 0) in drain_queue()
2827 pthread_mutex_unlock (&aq->mutex); in drain_queue()
2828 execute_queue_entry (aq, aq->queue_first); in drain_queue()
2830 pthread_mutex_lock (&aq->mutex); in drain_queue()
2831 aq->queue_first = ((aq->queue_first + 1) in drain_queue()
2833 aq->queue_n--; in drain_queue()
2837 aq->agent->device_id, aq->id); in drain_queue()
2838 pthread_cond_broadcast (&aq->queue_cond_out); in drain_queue()
2839 pthread_mutex_unlock (&aq->mutex); in drain_queue()
2842 GCN_DEBUG ("Async thread %d:%d: continue\n", aq->agent->device_id, in drain_queue()
2843 aq->id); in drain_queue()
2844 pthread_mutex_lock (&aq->mutex); in drain_queue()
2850 aq->agent->device_id, aq->id); in drain_queue()
2851 pthread_cond_wait (&aq->queue_cond_in, &aq->mutex); in drain_queue()
2854 aq->agent->device_id, aq->id); in drain_queue()
2858 aq->drain_queue_stop = 2; in drain_queue()
2861 aq->agent->device_id, aq->id); in drain_queue()
2862 pthread_cond_broadcast (&aq->queue_cond_out); in drain_queue()
2863 pthread_mutex_unlock (&aq->mutex); in drain_queue()
2865 GCN_DEBUG ("Async thread %d:%d: returning\n", aq->agent->device_id, aq->id); in drain_queue()
2873 drain_queue_synchronous (struct goacc_asyncqueue *aq) in drain_queue_synchronous() argument
2875 pthread_mutex_lock (&aq->mutex); in drain_queue_synchronous()
2877 while (aq->queue_n > 0) in drain_queue_synchronous()
2879 execute_queue_entry (aq, aq->queue_first); in drain_queue_synchronous()
2881 aq->queue_first = ((aq->queue_first + 1) in drain_queue_synchronous()
2883 aq->queue_n--; in drain_queue_synchronous()
2886 pthread_mutex_unlock (&aq->mutex); in drain_queue_synchronous()
2893 wait_for_queue_nonfull (struct goacc_asyncqueue *aq) in wait_for_queue_nonfull() argument
2895 if (aq->queue_n == ASYNC_QUEUE_SIZE) in wait_for_queue_nonfull()
2898 while (aq->queue_n == ASYNC_QUEUE_SIZE) in wait_for_queue_nonfull()
2899 pthread_cond_wait (&aq->queue_cond_out, &aq->mutex); in wait_for_queue_nonfull()
2908 queue_push_launch (struct goacc_asyncqueue *aq, struct kernel_info *kernel, in queue_push_launch() argument
2911 assert (aq->agent == kernel->agent); in queue_push_launch()
2913 pthread_mutex_lock (&aq->mutex); in queue_push_launch()
2915 wait_for_queue_nonfull (aq); in queue_push_launch()
2917 int queue_last = ((aq->queue_first + aq->queue_n) in queue_push_launch()
2920 GCN_DEBUG ("queue_push_launch %d:%d: at %i\n", aq->agent->device_id, in queue_push_launch()
2921 aq->id, queue_last); in queue_push_launch()
2923 aq->queue[queue_last].type = KERNEL_LAUNCH; in queue_push_launch()
2924 aq->queue[queue_last].u.launch.kernel = kernel; in queue_push_launch()
2925 aq->queue[queue_last].u.launch.vars = vars; in queue_push_launch()
2926 aq->queue[queue_last].u.launch.kla = *kla; in queue_push_launch()
2928 aq->queue_n++; in queue_push_launch()
2932 aq->agent->device_id, aq->id); in queue_push_launch()
2933 pthread_cond_signal (&aq->queue_cond_in); in queue_push_launch()
2935 pthread_mutex_unlock (&aq->mutex); in queue_push_launch()
2943 queue_push_callback (struct goacc_asyncqueue *aq, void (*fn)(void *), in queue_push_callback() argument
2946 pthread_mutex_lock (&aq->mutex); in queue_push_callback()
2948 wait_for_queue_nonfull (aq); in queue_push_callback()
2950 int queue_last = ((aq->queue_first + aq->queue_n) in queue_push_callback()
2953 GCN_DEBUG ("queue_push_callback %d:%d: at %i\n", aq->agent->device_id, in queue_push_callback()
2954 aq->id, queue_last); in queue_push_callback()
2956 aq->queue[queue_last].type = CALLBACK; in queue_push_callback()
2957 aq->queue[queue_last].u.callback.fn = fn; in queue_push_callback()
2958 aq->queue[queue_last].u.callback.data = data; in queue_push_callback()
2960 aq->queue_n++; in queue_push_callback()
2964 aq->agent->device_id, aq->id); in queue_push_callback()
2965 pthread_cond_signal (&aq->queue_cond_in); in queue_push_callback()
2967 pthread_mutex_unlock (&aq->mutex); in queue_push_callback()
2976 queue_push_asyncwait (struct goacc_asyncqueue *aq, in queue_push_asyncwait() argument
2979 pthread_mutex_lock (&aq->mutex); in queue_push_asyncwait()
2981 wait_for_queue_nonfull (aq); in queue_push_asyncwait()
2983 int queue_last = ((aq->queue_first + aq->queue_n) % ASYNC_QUEUE_SIZE); in queue_push_asyncwait()
2985 GCN_DEBUG ("queue_push_asyncwait %d:%d: at %i\n", aq->agent->device_id, in queue_push_asyncwait()
2986 aq->id, queue_last); in queue_push_asyncwait()
2988 aq->queue[queue_last].type = ASYNC_WAIT; in queue_push_asyncwait()
2989 aq->queue[queue_last].u.asyncwait.placeholderp = placeholderp; in queue_push_asyncwait()
2991 aq->queue_n++; in queue_push_asyncwait()
2995 aq->agent->device_id, aq->id); in queue_push_asyncwait()
2996 pthread_cond_signal (&aq->queue_cond_in); in queue_push_asyncwait()
2998 pthread_mutex_unlock (&aq->mutex); in queue_push_asyncwait()
3006 queue_push_placeholder (struct goacc_asyncqueue *aq) in queue_push_placeholder() argument
3010 pthread_mutex_lock (&aq->mutex); in queue_push_placeholder()
3012 wait_for_queue_nonfull (aq); in queue_push_placeholder()
3014 int queue_last = ((aq->queue_first + aq->queue_n) % ASYNC_QUEUE_SIZE); in queue_push_placeholder()
3016 GCN_DEBUG ("queue_push_placeholder %d:%d: at %i\n", aq->agent->device_id, in queue_push_placeholder()
3017 aq->id, queue_last); in queue_push_placeholder()
3019 aq->queue[queue_last].type = ASYNC_PLACEHOLDER; in queue_push_placeholder()
3020 placeholderp = &aq->queue[queue_last].u.placeholder; in queue_push_placeholder()
3024 pthread_mutex_unlock (&aq->mutex); in queue_push_placeholder()
3030 pthread_mutex_unlock (&aq->mutex); in queue_push_placeholder()
3036 aq->queue_n++; in queue_push_placeholder()
3040 aq->agent->device_id, aq->id); in queue_push_placeholder()
3041 pthread_cond_signal (&aq->queue_cond_in); in queue_push_placeholder()
3043 pthread_mutex_unlock (&aq->mutex); in queue_push_placeholder()
3051 finalize_async_thread (struct goacc_asyncqueue *aq) in finalize_async_thread() argument
3053 pthread_mutex_lock (&aq->mutex); in finalize_async_thread()
3054 if (aq->drain_queue_stop == 2) in finalize_async_thread()
3056 pthread_mutex_unlock (&aq->mutex); in finalize_async_thread()
3060 aq->drain_queue_stop = 1; in finalize_async_thread()
3064 aq->agent->device_id, aq->id); in finalize_async_thread()
3065 pthread_cond_signal (&aq->queue_cond_in); in finalize_async_thread()
3067 while (aq->drain_queue_stop != 2) in finalize_async_thread()
3071 " to sleep\n", aq->agent->device_id, aq->id); in finalize_async_thread()
3072 pthread_cond_wait (&aq->queue_cond_out, &aq->mutex); in finalize_async_thread()
3075 aq->agent->device_id, aq->id); in finalize_async_thread()
3078 GCN_DEBUG ("Done waiting for async thread %d:%d\n", aq->agent->device_id, in finalize_async_thread()
3079 aq->id); in finalize_async_thread()
3080 pthread_mutex_unlock (&aq->mutex); in finalize_async_thread()
3082 int err = pthread_join (aq->thread_drain_queue, NULL); in finalize_async_thread()
3085 aq->agent->device_id, aq->id, strerror (err)); in finalize_async_thread()
3086 GCN_DEBUG ("Joined with async thread %d:%d\n", aq->agent->device_id, aq->id); in finalize_async_thread()
3135 data->aq->agent->device_id, data->aq->id, data->len, data->src, in copy_data()
3157 queue_push_copy (struct goacc_asyncqueue *aq, void *dst, const void *src, in queue_push_copy() argument
3162 aq->agent->device_id, aq->id, len, src, dst); in queue_push_copy()
3169 data->aq = aq; in queue_push_copy()
3170 queue_push_callback (aq, copy_data, data); in queue_push_copy()
3176 queue_empty (struct goacc_asyncqueue *aq) in queue_empty() argument
3178 pthread_mutex_lock (&aq->mutex); in queue_empty()
3179 int res = aq->queue_n == 0 ? 1 : 0; in queue_empty()
3180 pthread_mutex_unlock (&aq->mutex); in queue_empty()
3189 wait_queue (struct goacc_asyncqueue *aq) in wait_queue() argument
3193 drain_queue_synchronous (aq); in wait_queue()
3197 pthread_mutex_lock (&aq->mutex); in wait_queue()
3199 while (aq->queue_n > 0) in wait_queue()
3203 aq->agent->device_id, aq->id); in wait_queue()
3204 pthread_cond_wait (&aq->queue_cond_out, &aq->mutex); in wait_queue()
3206 GCN_DEBUG ("thread %d:%d woke up. Rechecking\n", aq->agent->device_id, in wait_queue()
3207 aq->id); in wait_queue()
3210 pthread_mutex_unlock (&aq->mutex); in wait_queue()
3211 GCN_DEBUG ("waiting for thread %d:%d, done\n", aq->agent->device_id, aq->id); in wait_queue()
3222 struct goacc_asyncqueue *aq) in gcn_exec() argument
3335 queue_push_launch (aq, kernel, ind_da, &kla); in gcn_exec()
3338 aq->agent->device_id, aq->id, ind_da); in gcn_exec()
3339 queue_push_callback (aq, gomp_offload_free, ind_da); in gcn_exec()
3964 struct goacc_asyncqueue *aq) in GOMP_OFFLOAD_openacc_async_exec() argument
3969 aq); in GOMP_OFFLOAD_openacc_async_exec()
3981 struct goacc_asyncqueue *aq = GOMP_PLUGIN_malloc (sizeof (*aq)); in GOMP_OFFLOAD_openacc_async_construct() local
3982 aq->agent = get_agent_info (device); in GOMP_OFFLOAD_openacc_async_construct()
3983 aq->prev = NULL; in GOMP_OFFLOAD_openacc_async_construct()
3984 aq->next = agent->async_queues; in GOMP_OFFLOAD_openacc_async_construct()
3985 if (aq->next) in GOMP_OFFLOAD_openacc_async_construct()
3987 aq->next->prev = aq; in GOMP_OFFLOAD_openacc_async_construct()
3988 aq->id = aq->next->id + 1; in GOMP_OFFLOAD_openacc_async_construct()
3991 aq->id = 1; in GOMP_OFFLOAD_openacc_async_construct()
3992 agent->async_queues = aq; in GOMP_OFFLOAD_openacc_async_construct()
3994 aq->queue_first = 0; in GOMP_OFFLOAD_openacc_async_construct()
3995 aq->queue_n = 0; in GOMP_OFFLOAD_openacc_async_construct()
3996 aq->drain_queue_stop = 0; in GOMP_OFFLOAD_openacc_async_construct()
3998 if (pthread_mutex_init (&aq->mutex, NULL)) in GOMP_OFFLOAD_openacc_async_construct()
4003 if (pthread_cond_init (&aq->queue_cond_in, NULL)) in GOMP_OFFLOAD_openacc_async_construct()
4008 if (pthread_cond_init (&aq->queue_cond_out, NULL)) in GOMP_OFFLOAD_openacc_async_construct()
4019 &aq->hsa_queue); in GOMP_OFFLOAD_openacc_async_construct()
4023 int err = pthread_create (&aq->thread_drain_queue, NULL, &drain_queue, aq); in GOMP_OFFLOAD_openacc_async_construct()
4027 GCN_DEBUG ("Async thread %d:%d: created\n", aq->agent->device_id, in GOMP_OFFLOAD_openacc_async_construct()
4028 aq->id); in GOMP_OFFLOAD_openacc_async_construct()
4032 return aq; in GOMP_OFFLOAD_openacc_async_construct()
4039 GOMP_OFFLOAD_openacc_async_destruct (struct goacc_asyncqueue *aq) in GOMP_OFFLOAD_openacc_async_destruct() argument
4041 struct agent_info *agent = aq->agent; in GOMP_OFFLOAD_openacc_async_destruct()
4043 finalize_async_thread (aq); in GOMP_OFFLOAD_openacc_async_destruct()
4048 if ((err = pthread_mutex_destroy (&aq->mutex))) in GOMP_OFFLOAD_openacc_async_destruct()
4053 if (pthread_cond_destroy (&aq->queue_cond_in)) in GOMP_OFFLOAD_openacc_async_destruct()
4058 if (pthread_cond_destroy (&aq->queue_cond_out)) in GOMP_OFFLOAD_openacc_async_destruct()
4063 hsa_status_t status = hsa_fns.hsa_queue_destroy_fn (aq->hsa_queue); in GOMP_OFFLOAD_openacc_async_destruct()
4070 if (aq->prev) in GOMP_OFFLOAD_openacc_async_destruct()
4071 aq->prev->next = aq->next; in GOMP_OFFLOAD_openacc_async_destruct()
4072 if (aq->next) in GOMP_OFFLOAD_openacc_async_destruct()
4073 aq->next->prev = aq->prev; in GOMP_OFFLOAD_openacc_async_destruct()
4074 if (agent->async_queues == aq) in GOMP_OFFLOAD_openacc_async_destruct()
4075 agent->async_queues = aq->next; in GOMP_OFFLOAD_openacc_async_destruct()
4077 GCN_DEBUG ("Async thread %d:%d: destroyed\n", agent->device_id, aq->id); in GOMP_OFFLOAD_openacc_async_destruct()
4079 free (aq); in GOMP_OFFLOAD_openacc_async_destruct()
4091 GOMP_OFFLOAD_openacc_async_test (struct goacc_asyncqueue *aq) in GOMP_OFFLOAD_openacc_async_test() argument
4093 return queue_empty (aq); in GOMP_OFFLOAD_openacc_async_test()
4100 GOMP_OFFLOAD_openacc_async_synchronize (struct goacc_asyncqueue *aq) in GOMP_OFFLOAD_openacc_async_synchronize() argument
4102 wait_queue (aq); in GOMP_OFFLOAD_openacc_async_synchronize()
4127 GOMP_OFFLOAD_openacc_async_queue_callback (struct goacc_asyncqueue *aq, in GOMP_OFFLOAD_openacc_async_queue_callback() argument
4130 queue_push_callback (aq, fn, data); in GOMP_OFFLOAD_openacc_async_queue_callback()
4137 size_t n, struct goacc_asyncqueue *aq) in GOMP_OFFLOAD_openacc_async_host2dev() argument
4140 assert (agent == aq->agent); in GOMP_OFFLOAD_openacc_async_host2dev()
4149 queue_push_copy (aq, dst, src_copy, n, true); in GOMP_OFFLOAD_openacc_async_host2dev()
4157 size_t n, struct goacc_asyncqueue *aq) in GOMP_OFFLOAD_openacc_async_dev2host() argument
4160 assert (agent == aq->agent); in GOMP_OFFLOAD_openacc_async_dev2host()
4161 queue_push_copy (aq, dst, src, n, false); in GOMP_OFFLOAD_openacc_async_dev2host()