1 /*
2 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2017 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2006-2010 University of Houston. All rights reserved.
13 * Copyright (c) 2015-2016 Research Organization for Information Science
14 * and Technology (RIST). All rights reserved.
15 * $COPYRIGHT$
16 *
17 * Additional copyrights may follow
18 *
19 * $HEADER$
20 */
21
22 #include "ompi_config.h"
23 #include "coll_inter.h"
24
25 #include "mpi.h"
26 #include "ompi/constants.h"
27 #include "ompi/datatype/ompi_datatype.h"
28 #include "ompi/mca/coll/coll.h"
29 #include "ompi/mca/coll/base/coll_tags.h"
30 #include "ompi/mca/pml/pml.h"
31
32 /*
33 * gatherv_inter
34 *
35 * Function: - gatherv operation using a local gather on c_local_comm
36 * Accepts: - same arguments as MPI_Gatherv()
37 * Returns: - MPI_SUCCESS or error code
38 */
39 int
mca_coll_inter_gatherv_inter(const void * sbuf,int scount,struct ompi_datatype_t * sdtype,void * rbuf,const int * rcounts,const int * disps,struct ompi_datatype_t * rdtype,int root,struct ompi_communicator_t * comm,mca_coll_base_module_t * module)40 mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
41 struct ompi_datatype_t *sdtype,
42 void *rbuf, const int *rcounts, const int *disps,
43 struct ompi_datatype_t *rdtype, int root,
44 struct ompi_communicator_t *comm,
45 mca_coll_base_module_t *module)
46 {
47 int i, rank, size, size_local, total=0, err;
48 int *count=NULL, *displace=NULL;
49 char *ptmp_free=NULL, *ptmp=NULL;
50 ompi_datatype_t *ndtype;
51
52 if (MPI_PROC_NULL == root) { /* do nothing */
53 return OMPI_SUCCESS;
54 }
55 size = ompi_comm_remote_size(comm);
56 rank = ompi_comm_rank(comm);
57 size_local = ompi_comm_size(comm);
58
59 if (MPI_ROOT == root) { /* I am the root, receiving the data from zero. */
60 ompi_datatype_create_indexed(size, rcounts, disps, rdtype, &ndtype);
61 ompi_datatype_commit(&ndtype);
62
63 err = MCA_PML_CALL(recv(rbuf, 1, ndtype, 0,
64 MCA_COLL_BASE_TAG_GATHERV,
65 comm, MPI_STATUS_IGNORE));
66 ompi_datatype_destroy(&ndtype);
67 return err;
68 }
69
70 if (0 == rank) {
71 count = (int *)malloc(sizeof(int) * size_local);
72 displace = (int *)malloc(sizeof(int) * size_local);
73 if ((NULL == displace) || (NULL == count)) {
74 err = OMPI_ERR_OUT_OF_RESOURCE;
75 goto exit;
76 }
77 }
78
79 err = comm->c_local_comm->c_coll->coll_gather(&scount, 1, MPI_INT,
80 count, 1, MPI_INT,
81 0, comm->c_local_comm,
82 comm->c_local_comm->c_coll->coll_gather_module);
83 if (OMPI_SUCCESS != err) {
84 goto exit;
85 }
86 if(0 == rank) {
87 displace[0] = 0;
88 for (i = 1; i < size_local; i++) {
89 displace[i] = displace[i-1] + count[i-1];
90 }
91 /* Perform the gatherv locally with the first process as root */
92 for (i = 0; i < size_local; i++) {
93 total = total + count[i];
94 }
95 if ( total > 0 ) {
96 ptrdiff_t gap, span;
97 span = opal_datatype_span(&sdtype->super, total, &gap);
98 ptmp_free = (char*)malloc(span);
99 if (NULL == ptmp_free) {
100 err = OMPI_ERR_OUT_OF_RESOURCE;
101 goto exit;
102 }
103 ptmp = ptmp_free - gap;
104 }
105 }
106 err = comm->c_local_comm->c_coll->coll_gatherv(sbuf, scount, sdtype,
107 ptmp, count, displace,
108 sdtype,0, comm->c_local_comm,
109 comm->c_local_comm->c_coll->coll_gatherv_module);
110 if (OMPI_SUCCESS != err) {
111 goto exit;
112 }
113
114 if (0 == rank) {
115 /* First process sends data to the root */
116 err = MCA_PML_CALL(send(ptmp, total, sdtype, root,
117 MCA_COLL_BASE_TAG_GATHERV,
118 MCA_PML_BASE_SEND_STANDARD, comm));
119 }
120
121 exit:
122 if (NULL != ptmp_free) {
123 free(ptmp_free);
124 }
125 if (NULL != displace) {
126 free(displace);
127 }
128 if (NULL != count) {
129 free(count);
130 }
131
132 /* All done */
133 return err;
134 }
135