1 #include "BSprivate.h"
2
3 /*+ BSorig_inode - Change the inode structure to include the original,
4 unpermuted i-node numbers
5
6 Input Parameters:
7 . A - the sparse matrix
8 . procinfo - the usual processor stuff
9
10 Output Parameters:
11 . A - the sparse matrix changed to include the original i-node numbers
12
13 Returns:
14 void
15
16 +*/
BSorig_inode(BSpar_mat * A,BSprocinfo * procinfo)17 void BSorig_inode(BSpar_mat *A, BSprocinfo *procinfo)
18 {
19 BMcomp_msg *to_msg, *from_msg;
20 BMphase *phase_ptr, *to_phase, *from_phase;
21 BMmsg *msg;
22 int i, j, k;
23 int cl_ind, tcl_ind, cur_proc, in_ind;
24 int count, size;
25 int last;
26 int *setup_data, *user_data;
27 BScl_2_inode *clique2inode;
28 BSnumbering *color2clique;
29 BSinode *inodes;
30 int local_num;
31 int *msg_buf, *data_ptr, msg_len;
32 int *data, data_len, ind, *perm, *iperm;
33 void (*map)(int,int *,int *,BSprocinfo *,BSmapping *);
34 void (*lmap)(int,int *,int *,BSprocinfo *,BSmapping *);
35
36 /* initialize communication structures */
37 to_msg = BMcomp_init(COMP_MSG_BASE); CHKERR(0);
38 from_msg = BMcomp_init(COMP_MSG_BASE); CHKERR(0);
39
40 /* initialize variables for ease of use */
41 map = A->map->fglobal2local;
42 lmap = A->map->flocal2global;
43 perm = A->perm->perm;
44 iperm = A->inv_perm->perm;
45 color2clique = A->color2clique;
46 clique2inode = A->clique2inode;
47 inodes = A->inodes->list;
48
49 /* now go through and figure out everyone that we need stuff from */
50 /* do it by color, where each color is a phase */
51 for (i=0;i<color2clique->length-1;i++) {
52 cl_ind = color2clique->numbers[i];
53 while (cl_ind < color2clique->numbers[i+1]) {
54 /* figure out the message size and destination */
55 cur_proc = clique2inode->proc[cl_ind];
56 tcl_ind = cl_ind;
57 size = 0;
58 while ((tcl_ind < color2clique->numbers[i+1]) &&
59 (cur_proc == clique2inode->proc[tcl_ind])) {
60 /* add in the lengths, nonlocals are negative */
61 size += abs(clique2inode->d_mats[tcl_ind].size);
62 tcl_ind++;
63 }
64 if (cur_proc != procinfo->my_id) {
65 /* now we know the message size, allocate the message, etc */
66 msg = BMcreate_msg(i,-1,MPI_INT,cur_proc); CHKERR(0);
67 BMadd_msg(from_msg,msg,procinfo); CHKERR(0);
68 MY_MALLOC(user_data,(int *),sizeof(int)*2,1);
69 user_data[0] = cl_ind;
70 user_data[1] = tcl_ind-1;
71 BMset_user_data(msg,user_data,2,BSfree_comm_data);
72 CHKERR(0);
73 MY_MALLOC(setup_data,(int *),sizeof(int)*size,1);
74 count = 0;
75 while (count < size) {
76 for (j=clique2inode->inode_index[cl_ind];
77 j<clique2inode->inode_index[cl_ind+1];j++) {
78 setup_data[count] = inodes[j].o_gcol_num[0];
79 count++;
80 for (k=1;k<inodes[j].num_cols;k++) {
81 setup_data[count] = -1;
82 count++;
83 }
84 }
85 cl_ind++;
86 }
87 BMset_setup_data(msg,setup_data,size,BSfree_comm_data);
88 CHKERR(0);
89 BMset_msg_size(msg,size); CHKERR(0);
90 } else {
91 /* take care of local numbering */
92 for (cl_ind=cl_ind;cl_ind<tcl_ind;cl_ind++) {
93 for (j=clique2inode->inode_index[cl_ind];
94 j<clique2inode->inode_index[cl_ind+1];j++) {
95 (*map)(1,&(inodes[j].o_gcol_num[0]),&ind,procinfo,
96 A->map); CHKERR(0);
97 last = perm[ind]+1;
98 for (k=1;k<inodes[j].num_cols;k++) {
99 local_num = iperm[last];
100 (*lmap)(1,&local_num,&ind,procinfo,A->map);
101 CHKERR(0);
102 inodes[j].o_gcol_num[k] = ind;
103 last++;
104 }
105 }
106 }
107 }
108 }
109
110 /* now, let's work out what I need to send */
111 phase_ptr = BMget_phase(from_msg,i); CHKERR(0);
112 count = BMfix_send(SETUP_ORIGINAL_MSG,COMP_MSG_BASE,MPI_INT,to_msg,
113 phase_ptr,BSfree_comm_data,procinfo); CHKERR(0);
114
115 /* now free up the setup data */
116 msg = NULL;
117 while ((msg = BMnext_msg(phase_ptr,msg)) != NULL) {
118 CHKERR(0);
119 BMfree_setup_data(msg);
120 CHKERR(0);
121 }
122 CHKERR(0);
123 }
124
125 /* now, go over the messages that I have to send and translate the */
126 /* user data */
127 for (i=0;i<color2clique->length-1;i++) {
128 phase_ptr = BMget_phase(to_msg,i); CHKERR(0);
129 msg = NULL;
130 while ((msg = BMnext_msg(phase_ptr,msg)) != NULL) {
131 CHKERR(0);
132 data = BMget_user(msg,&data_len);
133 BMset_msg_size(msg,data_len); CHKERR(0);
134 last = -1;
135 for (j=0;j<data_len;j++) {
136 if (data[j] < 0) {
137 local_num = iperm[last];
138 (*lmap)(1,&local_num,&ind,procinfo,A->map); CHKERR(0);
139 data[j] = ind;
140 last++;
141 } else {
142 (*map)(1,&(data[j]),&ind,procinfo,A->map); CHKERR(0);
143 last = perm[ind]+1;
144 }
145 }
146 }
147 CHKERR(0);
148 }
149
150 /* organize the messages */
151 BMalloc_msg(to_msg); CHKERR(0);
152 BMalloc_msg(from_msg); CHKERR(0);
153
154 /* now send and receive the true info */
155 BMinit_comp_msg(from_msg,procinfo); CHKERR(0);
156 for (i=0;i<color2clique->length-1;i++) {
157 /* send messages */
158 to_phase = BMget_phase(to_msg,i); CHKERR(0);
159 msg = NULL;
160 while ((msg = BMnext_msg(to_phase,msg)) != NULL) {
161 CHKERR(0);
162 msg_buf = (int *) BMget_msg_ptr(msg); CHKERR(0);
163 data_ptr = BMget_user(msg,&msg_len); CHKERR(0);
164 for (j=0;j<msg_len;j++) {
165 msg_buf[j] = data_ptr[j];
166 }
167 BMsendf_msg(msg,procinfo); CHKERR(0);
168 }
169 CHKERR(0);
170
171 /* receive messages */
172 from_phase = BMget_phase(from_msg,i); CHKERR(0);
173 while ((msg = BMrecv_msg(from_phase)) != NULL) {
174 CHKERR(0);
175 msg_buf = (int *) BMget_msg_ptr(msg); CHKERR(0);
176 data_ptr = BMget_user(msg,&msg_len); CHKERR(0);
177 msg_len = BMget_msg_size(msg); CHKERR(0);
178 count = 0;
179 for (cl_ind=data_ptr[0];cl_ind<=data_ptr[1];cl_ind++) {
180 for (in_ind=clique2inode->inode_index[cl_ind];
181 in_ind<clique2inode->inode_index[cl_ind+1];in_ind++) {
182 for (j=0;j<inodes[in_ind].num_cols;j++) {
183 inodes[in_ind].o_gcol_num[j] = msg_buf[count];
184 count++;
185 }
186 }
187 }
188 BMfree_msg(msg); CHKERR(0);
189 }
190 CHKERR(0);
191 }
192
193 /* clean up the message stuff */
194 BMfinish_comp_msg(to_msg,procinfo); CHKERR(0);
195
196 /* free up the message structure */
197 BMfree_comp_msg(from_msg); CHKERR(0);
198 BMfree_comp_msg(to_msg); CHKERR(0);
199 }
200