1 /* makeSendRecvIVLs.c */
2
3 #include "../spoolesMPI.h"
4
5 /*--------------------------------------------------------------------*/
6 /*
7 -----------------------------------------------------------------
8 purpose -- to analyze and organize communication. it was written
9 in support of a distributed matrix-vector multiply but can be
10 used for other applications.
11
12 each processor has a list of items it "supports" or needs found
13 in the supportedIV object. the globalmapIV object contains the
14 map from items to owning processors. we need to figure out what
15 items this processor will send to and receive from each other
16 processor. this information is found in the sendIVL and recvIVL
17 objects.
18
19 on return, list jproc of sendIVL contains the items owned by
20 this processor and needed by jproc.
21 on return, list jproc of recvIVL contains the items needed by
22 this processor and owned by jproc.
23
24 as a concrete example, consider a distributed Y = A * X.
25 the matrix A, the right hand side X and the vector Y are
26 distributed among processors.
27
28 consider the case where the supportedIV object contains the rows
29 of X that are needed by this processor to perform its part of the
30 matrix-vector multiply. globalmapIV contains the map from rows
31 of X to the owning processors. on return, list jproc of sendIVL
32 contains the row indices of X owned by this processor that are
33 needed by processor jproc. on return, list jproc of recvIVL
34 contains the row indices of X needed by this processor that are
35 owned by processor jproc.
36
37 consider the case where the supportedIV object contains the rows
38 of Y that will be updated by this processor when it performs it
39 part of the matrix-vector multiply. globalmapIV contains the map
40 from rows of Y to their owning processors. on return, list jproc
41 of recvIVL contains the row indices of Y on this processor that
42 need to be sent to their owner, processor jproc. on return, list
43 jproc of sendIVL contains the row indices of Y owned by this
44 processor that will be sent by processor jproc to this processor.
45
46 created -- 98aug01, cca
47 -----------------------------------------------------------------
48 */
49 void
makeSendRecvIVLs(IV * supportedIV,IV * globalmapIV,IVL * sendIVL,IVL * recvIVL,int stats[],int msglvl,FILE * msgFile,int firsttag,MPI_Comm comm)50 makeSendRecvIVLs (
51 IV *supportedIV,
52 IV *globalmapIV,
53 IVL *sendIVL,
54 IVL *recvIVL,
55 int stats[],
56 int msglvl,
57 FILE *msgFile,
58 int firsttag,
59 MPI_Comm comm
60 ) {
61 int count, ii, item, jproc, maxitem, myid, nitem, nproc ;
62 int *head, *items, *link, *list, *map ;
63 /*
64 ---------------
65 check the input
66 ---------------
67 */
68 if ( supportedIV == NULL || globalmapIV == NULL
69 || sendIVL == NULL || recvIVL == NULL
70 || stats == NULL || (msglvl > 0 && msgFile == NULL) ) {
71 fprintf(stderr, "\n fatal error in makeSendRecvIVLs()"
72 "\n bad input\n") ;
73 exit(-1) ;
74 }
75 IV_sizeAndEntries(supportedIV, &nitem, &items) ;
76 if ( nitem == 0 ) {
77 maxitem = 0 ;
78 } else {
79 maxitem = items[nitem-1] ;
80 }
81 map = IV_entries(globalmapIV) ;
82 if ( msglvl > 2 ) {
83 fprintf(msgFile, "\n\n inside makeSendRecvIVLs()"
84 "\n supportedIV") ;
85 IV_writeForHumanEye(supportedIV, msgFile) ;
86 fprintf(msgFile, "\n globalmapIV") ;
87 IV_writeForHumanEye(globalmapIV, msgFile) ;
88 fflush(msgFile) ;
89 }
90 /*
91 ---------------------------------------
92 get id of self and number of processors
93 ---------------------------------------
94 */
95 MPI_Comm_rank(comm, &myid) ;
96 MPI_Comm_size(comm, &nproc) ;
97 /*
98 ----------------------------------------------------
99 link the items into lists via their owning processor
100 ----------------------------------------------------
101 */
102 head = IVinit(nproc, -1) ;
103 link = IVinit(1 + maxitem, -1) ;
104 for ( ii = 0 ; ii < nitem ; ii++ ) {
105 item = items[ii] ;
106 jproc = map[item] ;
107 link[item] = head[jproc] ;
108 head[jproc] = item ;
109 }
110 if ( msglvl > 2 ) {
111 fprintf(msgFile, "\n\n items linked by owning processor") ;
112 fflush(msgFile) ;
113 }
114 /*
115 ---------------------------------------
116 initialize and fill the recvIVL object.
117 list jproc contains the items that
118 this processor needs from jproc.
119 ---------------------------------------
120 */
121 IVL_init1(recvIVL, IVL_CHUNKED, nproc) ;
122 if ( nitem > 0 ) {
123 list = IVinit(nitem, -1) ;
124 for ( jproc = 0 ; jproc < nproc ; jproc++ ) {
125 count = 0 ;
126 for ( item = head[jproc] ; item != -1 ; item = link[item] ) {
127 list[count++] = item ;
128 }
129 IVqsortUp(count, list) ;
130 IVL_setList(recvIVL, jproc, count, list) ;
131 }
132 IVfree(list) ;
133 IVfree(head) ;
134 IVfree(link) ;
135 }
136 if ( msglvl > 5 ) {
137 fprintf(msgFile, "\n\n recvIVL") ;
138 IVL_writeForHumanEye(recvIVL, msgFile) ;
139 fflush(msgFile) ;
140 }
141 /*
142 ----------------------------------------------------------
143 compute the sendIVL object via an all-to-all communication
144 ----------------------------------------------------------
145 */
146 IVL_MPI_alltoall(recvIVL, sendIVL, stats,
147 msglvl, msgFile, firsttag, comm) ;
148
149 return ; }
150
151 /*--------------------------------------------------------------------*/
152