1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3  * Copyright (c) 2013      Mellanox Technologies, Inc.
4  *                         All rights reserved.
5  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
6  *                         reserved.
7  * Copyright (c) 2015      Research Organization for Information Science
8  *                         and Technology (RIST). All rights reserved.
9  * $COPYRIGHT$
10  *
11  * Additional copyrights may follow
12  *
13  * $HEADER$
14  */
15 
16 #include "oshmem_config.h"
17 
18 #include "ompi/communicator/communicator.h" /* TODO: ompi_predefined_communicator_t*/
19 #include "opal/class/opal_object.h"
20 #include "oshmem/request/request.h"
21 #include "oshmem/constants.h"
22 #include "oshmem/proc/proc.h"
23 
24 opal_pointer_array_t             oshmem_request_f_to_c_table = {{0}};
25 size_t                           oshmem_request_waiting = 0;
26 size_t                           oshmem_request_completed = 0;
27 opal_mutex_t                     oshmem_request_lock = {{0}};
28 opal_condition_t                 oshmem_request_cond = {{0}};
29 oshmem_predefined_request_t        oshmem_request_null = {{{{{0}}}}};
30 oshmem_request_t                   oshmem_request_empty = {{{{0}}}};
31 oshmem_status_public_t             oshmem_status_empty = {0};
32 oshmem_request_fns_t               oshmem_request_functions = {
33     NULL, /*oshmem_request_default_test,*/
34     NULL, /*oshmem_request_default_test_any,*/
35     NULL, /*oshmem_request_default_test_all,*/
36     NULL, /*oshmem_request_default_test_some,*/
37     NULL, /*oshmem_request_default_wait,*/
38     NULL, /*oshmem_request_default_wait_any,*/
39     NULL, /*oshmem_request_default_wait_all,*/
40     NULL, /*oshmem_request_default_wait_some*/
41 };
42 
oshmem_request_construct(oshmem_request_t * req)43 static void oshmem_request_construct(oshmem_request_t* req)
44 {
45     /* don't call _INIT, we don't to set the request to _INACTIVE and there will
46      * be no matching _FINI invocation */
47     req->req_state        = OSHMEM_REQUEST_INVALID;
48     req->req_complete     = false;
49     req->req_persistent   = false;
50     req->req_free = NULL;
51     req->req_cancel = NULL;
52     req->req_complete_cb = NULL;
53     req->req_complete_cb_data = NULL;
54     req->req_f_to_c_index = SHMEM_UNDEFINED;
55     req->req_shmem_object.comm = (oshmem_group_t*) NULL; /* TODO: Implement*/
56 }
57 
oshmem_request_destruct(oshmem_request_t * req)58 static void oshmem_request_destruct(oshmem_request_t* req)
59 {
60     assert( SHMEM_UNDEFINED == req->req_f_to_c_index);
61     assert( OSHMEM_REQUEST_INVALID == req->req_state);
62 }
63 
oshmem_request_null_free(oshmem_request_t ** request)64 static int oshmem_request_null_free(oshmem_request_t** request)
65 {
66     return OSHMEM_SUCCESS;
67 }
68 
oshmem_request_null_cancel(oshmem_request_t * request,int flag)69 static int oshmem_request_null_cancel(oshmem_request_t* request, int flag)
70 {
71     return OSHMEM_SUCCESS;
72 }
73 
oshmem_request_empty_free(oshmem_request_t ** request)74 static int oshmem_request_empty_free(oshmem_request_t** request)
75 {
76     *request = &oshmem_request_null.request;
77     return OSHMEM_SUCCESS;
78 }
79 
oshmem_request_persistent_proc_null_free(oshmem_request_t ** request)80 int oshmem_request_persistent_proc_null_free(oshmem_request_t** request)
81 {
82     OSHMEM_REQUEST_FINI(*request);
83     (*request)->req_state = OSHMEM_REQUEST_INVALID;
84     OBJ_RELEASE(*request);
85     *request = &oshmem_request_null.request;
86     return OSHMEM_SUCCESS;
87 }
88 
89 /*TODO: define under class oshmem_free_list_item_t */
90 OBJ_CLASS_INSTANCE( oshmem_request_t,
91                    opal_free_list_item_t,
92                    oshmem_request_construct,
93                    oshmem_request_destruct);
94 
oshmem_request_init(void)95 int oshmem_request_init(void)
96 {
97     OBJ_CONSTRUCT(&oshmem_request_lock, opal_mutex_t);
98     OBJ_CONSTRUCT(&oshmem_request_cond, opal_condition_t);
99 
100     OBJ_CONSTRUCT(&oshmem_request_null, oshmem_request_t);
101     OBJ_CONSTRUCT(&oshmem_request_f_to_c_table, opal_pointer_array_t);
102     if (OPAL_SUCCESS
103             != opal_pointer_array_init(&oshmem_request_f_to_c_table,
104                                        0,
105                                        OMPI_FORTRAN_HANDLE_MAX,
106                                        64)) {
107         return OSHMEM_ERROR;
108     }
109     oshmem_request_null.request.req_type = OSHMEM_REQUEST_NULL;
110     oshmem_request_null.request.req_status.SHMEM_SOURCE = SHMEM_PROC_NULL;
111     oshmem_request_null.request.req_status.SHMEM_ERROR = SHMEM_SUCCESS;
112     oshmem_request_null.request.req_status._count = 0;
113     oshmem_request_null.request.req_status._cancelled = 0;
114 
115     oshmem_request_null.request.req_complete = true;
116     oshmem_request_null.request.req_state = OSHMEM_REQUEST_INACTIVE;
117     oshmem_request_null.request.req_persistent = false;
118     oshmem_request_null.request.req_f_to_c_index =
119             opal_pointer_array_add(&oshmem_request_f_to_c_table,
120                                    &oshmem_request_null);
121     oshmem_request_null.request.req_free = oshmem_request_null_free;
122     oshmem_request_null.request.req_cancel = oshmem_request_null_cancel;
123     oshmem_request_null.request.req_shmem_object.comm =
124             (oshmem_group_t*) &ompi_mpi_comm_world.comm;
125 
126     if (0 != oshmem_request_null.request.req_f_to_c_index) {
127         return OSHMEM_ERR_REQUEST;
128     }
129 
130     /* We need a way to distinguish between the user provided
131      * SHMEM_REQUEST_NULL to SHMEM_Wait* and a non-active (SHMEM_PROC_NULL)
132      * request passed to any P2P non-blocking function.
133      *
134      * The main difference to oshmem_request_null is
135      * req_state being OSHMEM_REQUEST_ACTIVE, so that SHMEM_Waitall
136      * does not set the status to oshmem_status_empty and the different
137      * req_free function, which resets the
138      * request to SHMEM_REQUEST_NULL.
139      * The req_cancel function need not be changed.
140      */
141     OBJ_CONSTRUCT(&oshmem_request_empty, oshmem_request_t);
142     oshmem_request_empty.req_type = OSHMEM_REQUEST_NULL;
143     oshmem_request_empty.req_status.SHMEM_SOURCE = SHMEM_PROC_NULL;
144     oshmem_request_empty.req_status.SHMEM_ERROR = SHMEM_SUCCESS;
145     oshmem_request_empty.req_status._count = 0;
146     oshmem_request_empty.req_status._cancelled = 0;
147 
148     oshmem_request_empty.req_complete = true;
149     oshmem_request_empty.req_state = OSHMEM_REQUEST_ACTIVE;
150     oshmem_request_empty.req_persistent = false;
151     oshmem_request_empty.req_f_to_c_index =
152             opal_pointer_array_add(&oshmem_request_f_to_c_table,
153                                    &oshmem_request_empty);
154     oshmem_request_empty.req_free = oshmem_request_empty_free;
155     oshmem_request_empty.req_cancel = oshmem_request_null_cancel;
156     oshmem_request_empty.req_shmem_object.comm =
157             (oshmem_group_t*) &ompi_mpi_comm_world.comm;
158 
159     if (1 != oshmem_request_empty.req_f_to_c_index) {
160         return OSHMEM_ERR_REQUEST;
161     }
162 
163     oshmem_status_empty.SHMEM_SOURCE = SHMEM_ANY_SOURCE;
164     oshmem_status_empty.SHMEM_ERROR = SHMEM_SUCCESS;
165     oshmem_status_empty._count = 0;
166     oshmem_status_empty._cancelled = 0;
167 
168     return OSHMEM_SUCCESS;
169 }
170 
oshmem_request_finalize(void)171 int oshmem_request_finalize(void)
172 {
173     OSHMEM_REQUEST_FINI( &oshmem_request_null.request);
174     OBJ_DESTRUCT( &oshmem_request_null.request);
175     OSHMEM_REQUEST_FINI( &oshmem_request_empty);
176     OBJ_DESTRUCT( &oshmem_request_empty);
177     OBJ_DESTRUCT( &oshmem_request_cond);
178     OBJ_DESTRUCT( &oshmem_request_lock);
179     OBJ_DESTRUCT( &oshmem_request_f_to_c_table);
180     return OSHMEM_SUCCESS;
181 }
182