1 /* Optional support for MPI parallelization.
2 *
3 * Contents:
4 * 1. Communicating P7_HMM, a core model.
5 * 2. Communicating P7_PROFILE, a score profile.
6 * 3. Communicating P7_PIPELINE, pipeline stats.
7 * 4. Communicating P7_TOPHITS, list of high scoring alignments.
8 * 5. Benchmark driver.
9 * 6. Unit tests.
10 * 7. Test driver.
11 */
12 #include "p7_config.h"
13
14 #ifdef HMMER_MPI
15 #include <stdlib.h>
16 #include <stdio.h>
17 #include <string.h>
18
19 #include "mpi.h"
20
21 #include "easel.h"
22 #include "esl_mpi.h"
23 #include "esl_getopts.h"
24
25 #include "hmmer.h"
26
27 static int p7_hit_MPISend(P7_HIT *hit, int dest, int tag, MPI_Comm comm, char **buf, int *nalloc);
28 static int p7_hit_MPIPackSize(P7_HIT *hit, MPI_Comm comm, int *ret_n);
29 static int p7_hit_MPIPack(P7_HIT *hit, char *buf, int n, int *pos, MPI_Comm comm);
30 static int p7_hit_MPIUnpack(char *buf, int n, int *pos, MPI_Comm comm, P7_HIT *hit);
31 static int p7_hit_MPIRecv(int source, int tag, MPI_Comm comm, char **buf, int *nalloc, P7_HIT *hit);
32
33 static int p7_dcl_MPISend(P7_DOMAIN *dcl, int dest, int tag, MPI_Comm comm, char **buf, int *nalloc);
34 static int p7_dcl_MPIPackSize(P7_DOMAIN *dcl, MPI_Comm comm, int *ret_n);
35 static int p7_dcl_MPIPack(P7_DOMAIN *dcl, char *buf, int n, int *pos, MPI_Comm comm);
36 static int p7_dcl_MPIUnpack(char *buf, int n, int *pos, MPI_Comm comm, P7_DOMAIN *dcl);
37 static int p7_dcl_MPIRecv(int source, int tag, MPI_Comm comm, char **buf, int *nalloc, P7_DOMAIN *dcl);
38
39 /*****************************************************************
40 * 1. Communicating P7_HMM, a core model.
41 *****************************************************************/
42
43 /* Function: p7_hmm_MPISend()
44 * Synopsis: Send an HMM as an MPI work unit.
45 *
46 * Purpose: Sends an HMM <hmm> as a work unit to MPI process
47 * <dest> (where <dest> ranges from 0..<nproc-1>), tagged
48 * with MPI tag <tag>, for MPI communicator <comm>, as
49 * the sole workunit or result.
50 *
51 * Work units are prefixed by a status code. If <hmm> is
52 * <non-NULL>, the work unit is an <eslOK> code followed by
53 * the packed HMM. If <hmm> is NULL, the work unit is an
54 * <eslEOD> code, which <p7_hmm_MPIRecv()> knows how to
55 * interpret; this is typically used for an end-of-data
56 * signal to cleanly shut down worker processes.
57 *
58 * In order to minimize alloc/free cycles in this routine,
59 * caller passes a pointer to a working buffer <*buf> of
60 * size <*nalloc> characters. If necessary (i.e. if <hmm> is
61 * too big to fit), <*buf> will be reallocated and <*nalloc>
62 * increased to the new size. As a special case, if <*buf>
63 * is <NULL> and <*nalloc> is 0, the buffer will be
64 * allocated appropriately, but the caller is still
65 * responsible for free'ing it.
66 *
67 * Returns: <eslOK> on success; <*buf> may have been reallocated and
68 * <*nalloc> may have been increased.
69 *
70 * Throws: <eslESYS> if an MPI call fails; <eslEMEM> if a malloc/realloc
71 * fails. In either case, <*buf> and <*nalloc> remain valid and useful
72 * memory (though the contents of <*buf> are undefined).
73 *
74 * Note: Compare to p7_hmmfile_WriteBinary(). The two operations (sending
75 * an HMM via MPI, or saving it as a binary file to disk) are
76 * similar.
77 */
78 int
p7_hmm_MPISend(P7_HMM * hmm,int dest,int tag,MPI_Comm comm,char ** buf,int * nalloc)79 p7_hmm_MPISend(P7_HMM *hmm, int dest, int tag, MPI_Comm comm, char **buf, int *nalloc)
80 {
81 int status;
82 int code;
83 int sz, n, pos;
84
85 /* Figure out size */
86 if (MPI_Pack_size(1, MPI_INT, comm, &n) != 0) ESL_XEXCEPTION(eslESYS, "mpi pack size failed");
87 if (hmm != NULL) {
88 if ((status = p7_hmm_MPIPackSize(hmm, comm, &sz)) != eslOK) return status;
89 n += sz;
90 }
91
92 /* Make sure the buffer is allocated appropriately */
93 if (*buf == NULL || n > *nalloc) {
94 void *tmp;
95 ESL_RALLOC(*buf, tmp, sizeof(char) * n);
96 *nalloc = n;
97 }
98
99 /* Pack the status code and HMM into the buffer */
100 pos = 0;
101 code = (hmm == NULL) ? eslEOD : eslOK;
102 if (MPI_Pack(&code, 1, MPI_INT, *buf, n, &pos, comm) != 0) ESL_EXCEPTION(eslESYS, "mpi pack failed");
103 if (hmm != NULL) {
104 if ((status = p7_hmm_MPIPack(hmm, *buf, n, &pos, comm)) != eslOK) return status;
105 }
106
107 /* Send the packed HMM to the destination. */
108 if (MPI_Send(*buf, n, MPI_PACKED, dest, tag, comm) != 0) ESL_EXCEPTION(eslESYS, "mpi send failed");
109 return eslOK;
110
111 ERROR:
112 return status;
113 }
114
115 /* Function: p7_hmm_MPIPackSize()
116 * Synopsis: Calculates size needed to pack an HMM.
117 *
118 * Purpose: Calculate an upper bound on the number of bytes
119 * that <p7_hmm_MPIPack()> will need to pack an HMM
120 * <hmm> in a packed MPI message for MPI communicator
121 * <comm>; return that number of bytes in <*ret_n>.
122 *
123 * Returns: <eslOK> on success, and <*ret_n> contains the answer.
124 *
125 * Throws: <eslESYS> if an MPI call fails, and <*ret_n> is 0.
126 */
127 int
p7_hmm_MPIPackSize(P7_HMM * hmm,MPI_Comm comm,int * ret_n)128 p7_hmm_MPIPackSize(P7_HMM *hmm, MPI_Comm comm, int *ret_n)
129 {
130 int status;
131 int n = 0;
132 int K = hmm->abc->K;
133 int M = hmm->M;
134 int sz;
135
136 if (MPI_Pack_size(1, MPI_INT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += 6*sz; /* M,flags,nseq,eff_nseq,checksum,alphatype */
137 if (MPI_Pack_size(1, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += 6*sz; /* ga,tc,nc cutoffs */
138 if (MPI_Pack_size(7*(M+1), MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* t */
139 if (MPI_Pack_size(K*(M+1), MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += 2*sz; /* mat,ins */
140
141 if ((status = esl_mpi_PackOptSize(hmm->name, -1, MPI_CHAR, comm, &sz)) != eslOK) goto ERROR; else n += sz;
142 if ((status = esl_mpi_PackOptSize(hmm->acc, -1, MPI_CHAR, comm, &sz)) != eslOK) goto ERROR; else n += sz;
143 if ((status = esl_mpi_PackOptSize(hmm->desc, -1, MPI_CHAR, comm, &sz)) != eslOK) goto ERROR; else n += sz;
144
145 if (hmm->flags & p7H_RF) { if (MPI_Pack_size(M+2, MPI_CHAR, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n+= sz; }
146 if (hmm->flags & p7H_MMASK) { if (MPI_Pack_size(M+2, MPI_CHAR, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n+= sz; }
147 if (hmm->flags & p7H_CONS) { if (MPI_Pack_size(M+2, MPI_CHAR, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n+= sz; }
148 if (hmm->flags & p7H_CS) { if (MPI_Pack_size(M+2, MPI_CHAR, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n+= sz; }
149 if (hmm->flags & p7H_CA) { if (MPI_Pack_size(M+2, MPI_CHAR, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n+= sz; }
150 if ((status = esl_mpi_PackOptSize(hmm->comlog, -1, MPI_CHAR, comm, &sz)) != eslOK) goto ERROR; else n+= sz;
151 if ((status = esl_mpi_PackOptSize(hmm->ctime, -1, MPI_CHAR, comm, &sz)) != eslOK) goto ERROR; else n+= sz;
152 if (hmm->flags & p7H_MAP) { if (MPI_Pack_size(M+1, MPI_INT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n+= sz; }
153 if (MPI_Pack_size(p7_NEVPARAM, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n+= sz;
154 if (MPI_Pack_size(p7_NCUTOFFS, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n+= sz;
155 if (MPI_Pack_size(p7_MAXABET, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n+= sz;
156 *ret_n = n;
157 return eslOK;
158
159 ERROR:
160 *ret_n = 0;
161 return status;
162
163 }
164
165 /* Function: p7_hmm_MPIPack()
166 * Synopsis: Packs an HMM into MPI buffer.
167 *
168 * Purpose: Packs HMM <hmm> into an MPI packed message buffer <buf>
169 * of length <n> bytes, starting at byte position <*position>,
170 * for MPI communicator <comm>.
171 *
172 * The caller must know that <buf>'s allocation of <n>
173 * bytes is large enough to append the packed HMM at
174 * position <*pos>. This typically requires a call to
175 * <p7_hmm_MPIPackSize()> first, and reallocation if
176 * needed.
177 *
178 * Returns: <eslOK> on success; <buf> now contains the
179 * packed <hmm>, and <*position> is set to the byte
180 * immediately following the last byte of the HMM
181 * in <buf>.
182 *
183 * Throws: <eslESYS> if an MPI call fails; or <eslEMEM> if the
184 * buffer's length <n> was overflowed in trying to pack
185 * <msa> into <buf>. In either case, the state of
186 * <buf> and <*position> is undefined, and both should
187 * be considered to be corrupted.
188 */
189 int
p7_hmm_MPIPack(P7_HMM * hmm,char * buf,int n,int * pos,MPI_Comm comm)190 p7_hmm_MPIPack(P7_HMM *hmm, char *buf, int n, int *pos, MPI_Comm comm)
191 {
192 int status;
193 int K = hmm->abc->K;
194 int M = hmm->M;
195
196 if (MPI_Pack( &M, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
197 if (MPI_Pack( &(hmm->flags), 1, MPI_INT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
198 if (MPI_Pack( (void *) &(hmm->abc->type), 1, MPI_INT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
199 if (MPI_Pack( hmm->t[0], 7*(M+1), MPI_FLOAT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
200 if (MPI_Pack( hmm->mat[0], K*(M+1), MPI_FLOAT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
201 if (MPI_Pack( hmm->ins[0], K*(M+1), MPI_FLOAT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
202
203 if ((status = esl_mpi_PackOpt( hmm->name, -1, MPI_CHAR, buf, n, pos, comm)) != eslOK) return status;
204 if ((status = esl_mpi_PackOpt( hmm->acc, -1, MPI_CHAR, buf, n, pos, comm)) != eslOK) return status;
205 if ((status = esl_mpi_PackOpt( hmm->desc, -1, MPI_CHAR, buf, n, pos, comm)) != eslOK) return status;
206 if (hmm->flags & p7H_RF) { if (MPI_Pack(hmm->rf, M+2, MPI_CHAR, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed"); }
207 if (hmm->flags & p7H_MMASK) { if (MPI_Pack(hmm->mm, M+2, MPI_CHAR, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed"); }
208 if (hmm->flags & p7H_CONS) { if (MPI_Pack(hmm->consensus, M+2, MPI_CHAR, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed"); }
209 if (hmm->flags & p7H_CS) { if (MPI_Pack(hmm->cs, M+2, MPI_CHAR, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed"); }
210 if (hmm->flags & p7H_CA) { if (MPI_Pack(hmm->ca, M+2, MPI_CHAR, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed"); }
211 if ((status = esl_mpi_PackOpt( hmm->comlog, -1, MPI_CHAR, buf, n, pos, comm)) != eslOK) return status;
212 if (MPI_Pack( &(hmm->nseq), 1, MPI_INT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
213 if (MPI_Pack( &(hmm->eff_nseq), 1, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
214 if ((status = esl_mpi_PackOpt( hmm->ctime, -1, MPI_CHAR, buf, n, pos, comm)) != eslOK) return status;
215 if (hmm->flags & p7H_MAP) { if (MPI_Pack(hmm->map, M+1, MPI_INT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed"); }
216 if (MPI_Pack( &(hmm->checksum), 1, MPI_INT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
217 if (MPI_Pack( hmm->evparam, p7_NEVPARAM, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
218 if (MPI_Pack( hmm->cutoff, p7_NCUTOFFS, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
219 if (MPI_Pack( hmm->compo, p7_MAXABET, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_EXCEPTION(eslESYS, "pack failed");
220
221 if (*pos > n) ESL_EXCEPTION(eslEMEM, "buffer overflow");
222 return eslOK;
223 }
224
225
226 /* Function: p7_hmm_MPIUnpack()
227 * Synopsis: Unpacks an HMM from an MPI buffer.
228 *
229 * Purpose: Unpack a newly allocated HMM from MPI packed buffer
230 * <buf>, starting from position <*pos>, where the total length
231 * of the buffer in bytes is <n>.
232 *
233 * Caller may or may not already know what alphabet the HMM
234 * is expected to be in. A reference to the current
235 * alphabet is passed in <abc>. If the alphabet is unknown,
236 * pass <*abc = NULL>, and when the HMM is received, an
237 * appropriate new alphabet object is allocated and passed
238 * back to the caller via <*abc>. If the alphabet is
239 * already known, <*abc> is that alphabet, and the new
240 * HMM's alphabet type is verified to agree with it. This
241 * mechanism allows an application to let the first HMM
242 * determine the alphabet type for the application, while
243 * still keeping the alphabet under the application's scope
244 * of control.
245 *
246 * Returns: <eslOK> on success. <*pos> is updated to the position of
247 * the next element in <buf> to unpack (if any). <*ret_hmm>
248 * contains a newly allocated HMM, which the caller is
249 * responsible for free'ing. If <*abc> was passed as
250 * <NULL>, it now points to an <ESL_ALPHABET> object that
251 * was allocated here; caller is responsible for free'ing
252 * this.
253 *
254 * Returns <eslEINCOMPAT> if the HMM is in a different
255 * alphabet than <*abc> said to expect. In this case,
256 * <*abc> is unchanged, <*buf> and <*nalloc> may have been
257 * changed, and <*ret_hmm> is <NULL>.
258 *
259 * Throws: <eslESYS> on an MPI call failure. <eslEMEM> on allocation failure.
260 * In either case, <*ret_hmm> is <NULL>, and the state of <buf>
261 * and <*pos> is undefined and should be considered to be corrupted.
262 */
263 int
p7_hmm_MPIUnpack(char * buf,int n,int * pos,MPI_Comm comm,ESL_ALPHABET ** abc,P7_HMM ** ret_hmm)264 p7_hmm_MPIUnpack(char *buf, int n, int *pos, MPI_Comm comm, ESL_ALPHABET **abc, P7_HMM **ret_hmm)
265 {
266 int status;
267 P7_HMM *hmm = NULL;
268 int M, K, atype;
269
270 if ((hmm = p7_hmm_CreateShell()) == NULL) { status = eslEMEM; goto ERROR; }
271 if (MPI_Unpack(buf, n, pos, &(hmm->M), 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
272 if (MPI_Unpack(buf, n, pos, &(hmm->flags), 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
273 if (MPI_Unpack(buf, n, pos, &atype, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
274
275 /* Set or verify the alphabet */
276 if (*abc == NULL) { /* still unknown: set it, pass control of it back to caller */
277 if ((*abc = esl_alphabet_Create(atype)) == NULL) { status = eslEMEM; goto ERROR; }
278 } else { /* already known: check it */
279 if ((*abc)->type != atype) { status = eslEINCOMPAT; goto ERROR; }
280 }
281
282 /* For convenience below. */
283 K = (*abc)->K;
284 M = hmm->M;
285
286 /* Finish the allocation of the HMM */
287 if ((status = p7_hmm_CreateBody(hmm, M, *abc)) != eslOK) goto ERROR;
288
289 /* Unpack the rest of the HMM */
290 if (MPI_Unpack( buf, n, pos, hmm->t[0], 7*(M+1), MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
291 if (MPI_Unpack( buf, n, pos, hmm->mat[0], K*(M+1), MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
292 if (MPI_Unpack( buf, n, pos, hmm->ins[0], K*(M+1), MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
293 if ((status = esl_mpi_UnpackOpt( buf, n, pos, (void**)&(hmm->name), NULL, MPI_CHAR, comm)) != eslOK) goto ERROR;
294 if ((status = esl_mpi_UnpackOpt( buf, n, pos, (void**)&(hmm->acc), NULL, MPI_CHAR, comm)) != eslOK) goto ERROR;
295 if ((status = esl_mpi_UnpackOpt( buf, n, pos, (void**)&(hmm->desc), NULL, MPI_CHAR, comm)) != eslOK) goto ERROR;
296 if (hmm->flags & p7H_RF) { if (MPI_Unpack(buf, n, pos, hmm->rf, M+2, MPI_CHAR, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed"); }
297 if (hmm->flags & p7H_MMASK) { if (MPI_Unpack(buf, n, pos, hmm->mm, M+2, MPI_CHAR, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed"); }
298 if (hmm->flags & p7H_CONS) { if (MPI_Unpack(buf, n, pos, hmm->consensus, M+2, MPI_CHAR, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed"); }
299 if (hmm->flags & p7H_CS) { if (MPI_Unpack(buf, n, pos, hmm->cs, M+2, MPI_CHAR, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed"); }
300 if (hmm->flags & p7H_CA) { if (MPI_Unpack(buf, n, pos, hmm->ca, M+2, MPI_CHAR, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed"); }
301 if ((status = esl_mpi_UnpackOpt( buf, n, pos, (void**)&(hmm->comlog), NULL, MPI_CHAR, comm)) != eslOK) goto ERROR;
302 if (MPI_Unpack( buf, n, pos, &(hmm->nseq), 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
303 if (MPI_Unpack( buf, n, pos, &(hmm->eff_nseq), 1, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
304 if ((status = esl_mpi_UnpackOpt( buf, n, pos, (void**)&(hmm->ctime), NULL, MPI_CHAR, comm)) != eslOK) goto ERROR;
305 if (hmm->flags & p7H_MAP) { if (MPI_Unpack(buf, n, pos, hmm->map, M+1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed"); }
306 if (MPI_Unpack( buf, n, pos, &(hmm->checksum), 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
307 if (MPI_Unpack( buf, n, pos, hmm->evparam, p7_NEVPARAM, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
308 if (MPI_Unpack( buf, n, pos, hmm->cutoff, p7_NCUTOFFS, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
309 if (MPI_Unpack( buf, n, pos, hmm->compo, p7_MAXABET, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
310
311 *ret_hmm = hmm;
312 return eslOK;
313
314 ERROR:
315 if (hmm != NULL) p7_hmm_Destroy(hmm);
316 return status;
317 }
318
319
320
321
322 /* Function: p7_hmm_MPIRecv()
323 * Synopsis: Receives an HMM as a work unit from an MPI sender.
324 *
325 * Purpose: Receive a work unit that consists of a single HMM
326 * sent by MPI <source> (<0..nproc-1>, or
327 * <MPI_ANY_SOURCE>) tagged as <tag> for MPI communicator <comm>.
328 *
329 * Work units are prefixed by a status code. If the unit's
330 * code is <eslOK> and no errors are encountered, this
331 * routine will return <eslOK> and a non-<NULL> <*ret_hmm>.
332 * If the unit's code is <eslEOD> (a shutdown signal),
333 * this routine returns <eslEOD> and <*ret_hmm> is <NULL>.
334 *
335 * Caller provides a working buffer <*buf> of size
336 * <*nalloc> characters. These are passed by reference, so
337 * that <*buf> can be reallocated and <*nalloc> increased
338 * if necessary. As a special case, if <*buf> is <NULL> and
339 * <*nalloc> is 0, the buffer will be allocated
340 * appropriately, but the caller is still responsible for
341 * free'ing it.
342 *
343 * Caller may or may not already know what alphabet the HMM
344 * is expected to be in. A reference to the current
345 * alphabet is passed in <abc>. If the alphabet is unknown,
346 * pass <*abc = NULL>, and when the HMM is received, an
347 * appropriate new alphabet object is allocated and passed
348 * back to the caller via <*abc>. If the alphabet is
349 * already known, <*ret_abc> is that alphabet, and the new
350 * HMM's alphabet type is verified to agree with it. This
351 * mechanism allows an application to let the first HMM
352 * determine the alphabet type for the application, while
353 * still keeping the alphabet under the application's scope
354 * of control.
355 *
356 * Returns: <eslOK> on success. <*ret_hmm> contains the received HMM;
357 * it is allocated here, and the caller is responsible for
358 * free'ing it. <*buf> may have been reallocated to a
359 * larger size, and <*nalloc> may have been increased. If
360 * <*abc> was passed as <NULL>, it now points to an
361 * <ESL_ALPHABET> object that was allocated here; caller is
362 * responsible for free'ing this.
363 *
364 * Returns <eslEOD> if an end-of-data signal was received.
365 * In this case, <*buf>, <*nalloc>, and <*abc> are left unchanged,
366 * and <*ret_hmm> is <NULL>.
367 *
368 * Returns <eslEINCOMPAT> if the HMM is in a different alphabet
369 * than <*abc> said to expect. In this case, <*abc> is unchanged,
370 * <*buf> and <*nalloc> may have been changed, and <*ret_hmm> is
371 * <NULL>.
372 *
373 * Throws: <eslEMEM> on allocation error, in which case <*ret_hmm> is
374 * <NULL>.
375 */
376 int
p7_hmm_MPIRecv(int source,int tag,MPI_Comm comm,char ** buf,int * nalloc,ESL_ALPHABET ** abc,P7_HMM ** ret_hmm)377 p7_hmm_MPIRecv(int source, int tag, MPI_Comm comm, char **buf, int *nalloc, ESL_ALPHABET **abc, P7_HMM **ret_hmm)
378 {
379 int status;
380 int code;
381 P7_HMM *hmm = NULL;
382 int n;
383 int pos;
384 MPI_Status mpistatus;
385
386 /* Probe first, because we need to know if our buffer is big enough. */
387 MPI_Probe(source, tag, comm, &mpistatus);
388 MPI_Get_count(&mpistatus, MPI_PACKED, &n);
389
390 /* Make sure the buffer is allocated appropriately */
391 if (*buf == NULL || n > *nalloc) {
392 void *tmp;
393 ESL_RALLOC(*buf, tmp, sizeof(char) * n);
394 *nalloc = n;
395 }
396
397 /* Receive the packed work unit */
398 MPI_Recv(*buf, n, MPI_PACKED, source, tag, comm, &mpistatus);
399
400 /* Unpack it, looking at the status code prefix for EOD/EOK */
401 pos = 0;
402 if (MPI_Unpack(*buf, n, &pos, &code, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
403 if (code == eslEOD) { *ret_hmm = NULL; return eslEOD; }
404
405 return p7_hmm_MPIUnpack(*buf, *nalloc, &pos, comm, abc, ret_hmm);
406
407 ERROR:
408 if (hmm != NULL) p7_hmm_Destroy(hmm);
409 return status;
410 }
411
412 /*----------------- end, P7_HMM communication -------------------*/
413
414
415 /*****************************************************************
416 * 2. Communicating P7_PROFILE, a score profile.
417 *****************************************************************/
418
419 /* Function: p7_profile_MPISend()
420 * Synopsis: Send a profile as an MPI message.
421 *
422 * Purpose: Sends profile <gm> to MPI process <dest> (where
423 * <dest> ranges from 0..<nproc-1>), with MPI tag <tag>
424 * for MPI communicator <comm>.
425 *
426 * In order to minimize alloc/free cycles in this routine,
427 * caller passes a pointer to a working buffer <*buf> of
428 * size <*nalloc> characters. If necessary (i.e. if <gm> is
429 * too big to fit), <*buf> will be reallocated and <*nalloc>
430 * increased to the new size. As a special case, if <*buf>
431 * is <NULL> and <*nalloc> is 0, the buffer will be
432 * allocated appropriately, but the caller is still
433 * responsible for free'ing it.
434 *
435 * If <gm> is NULL, an end-of-data signal is sent, which
436 * <p7_profile_MPIRecv()> knows how to interpret.
437 *
438 * Returns: <eslOK> on success.
439 *
440 * Note: This was tested against a version that simply issues a series
441 * of MPI_Send()'s, rather than Pack()'ing into a buffer
442 * and issuing one MPI_Send(). The packed version seems to
443 * be significantly faster, although benchmarking MPI
444 * programs is difficult, and variance on the results is high.
445 *
446 * To optimize communication still further, one might try
447 * to avoid many or all of the MPI_Pack()'s. It might be
448 * feasible to change the allocation of a profile such that
449 * it is allocated in one contiguous chunk of memory. And
450 * once one embarks on that, the memory layout of the
451 * profile should also be optimized with respect to cache
452 * performance during DP alignment.
453 *
454 * rf, cs annotation is optional, but for simplicity, we always
455 * transmit the two allocated strings, even if they were empty.
456 *
457 * A "typical" protein profile (M=200,Kp=28) requires:
458 * 8*200 floats for transitions = 1600 bytes
459 * 28*201*2 floats for emissions = 11256 bytes
460 * 8 floats for specials = 32 bytes
461 * 2 ints for M,mode = 8 bytes
462 * 1 float for nj = 4 bytes
463 * ~100 chars of name, acc, desc = 100 bytes
464 * 3*202 chars for annotation = 606 bytes
465 * 3 floats for ev params = 12 bytes
466 * 6 floats for Pfam cutoffs = 24 bytes
467 * -----------
468 * 14Kb
469 */
470 int
p7_profile_MPISend(P7_PROFILE * gm,int dest,int tag,MPI_Comm comm,char ** buf,int * nalloc)471 p7_profile_MPISend(P7_PROFILE *gm, int dest, int tag, MPI_Comm comm, char **buf, int *nalloc)
472 {
473 int status;
474 int sz, n, position;
475 int Kp; /* alphabet size including degeneracies */
476 int M; /* model size in nodes */
477
478
479
480 /* First, figure out the size of the profile */
481 if (gm == NULL) {
482 if (MPI_Pack_size(1, MPI_INT, comm, &n) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed");
483 Kp = M = 0;
484 } else {
485 /* This will look wasteful, but the MPI spec doesn't guarantee that
486 * MPI_Pack_size(x, ...) + MPI_Pack_size(x, ... ) == MPI_Pack_size(2x, ...).
487 * Indeed there are some hints in the spec that that's *not* true.
488 * So we assume we must match our Pack_size calls exactly to our Pack calls.
489 */
490 Kp = gm->abc->Kp;
491 M = gm->M;
492 n = 0;
493 if (MPI_Pack_size( 1, MPI_INT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n += sz*3; /* M,mode,L */
494 if (MPI_Pack_size( p7P_NTRANS * M, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n += sz; /* tsc */
495 if (MPI_Pack_size( (M+1) * Kp * p7P_NR, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n += sz; /* rsc[0] */
496 if (MPI_Pack_size( p7P_NXTRANS, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n += sz*p7P_NXSTATES; /* xsc[0..3] */
497 if (MPI_Pack_size( 1, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n += sz; /* nj */
498 if ((status = esl_mpi_PackOptSize(gm->name, -1, MPI_CHAR, comm, &sz))!= eslOK) goto ERROR; else n += sz; /* name (string) */
499 if ((status = esl_mpi_PackOptSize(gm->acc, -1, MPI_CHAR, comm, &sz))!= eslOK) goto ERROR; else n += sz; /* acc (string) */
500 if ((status = esl_mpi_PackOptSize(gm->desc, -1, MPI_CHAR, comm, &sz))!= eslOK) goto ERROR; else n += sz; /* desc (string) */
501 if (MPI_Pack_size( (M+2), MPI_CHAR, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n += sz*4; /* rf,cs,mm,consensus */
502 if (MPI_Pack_size( p7_NEVPARAM, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n += sz; /* evparam */
503 if (MPI_Pack_size( p7_NCUTOFFS, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); else n += sz; /* Pfam cutoffs */
504 }
505
506 /* Make sure the buffer is allocated appropriately */
507 if (*buf == NULL || n > *nalloc) {
508 void *tmp;
509 ESL_RALLOC(*buf, tmp, sizeof(char) * n);
510 *nalloc = n;
511 }
512
513 /* Pack the profile into the buffer */
514 position = 0;
515 if (gm == NULL)
516 {
517 int eod_code = -1;
518 if (MPI_Pack(&eod_code, 1, MPI_INT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
519 }
520 else
521 {
522 if (MPI_Pack(&M, 1, MPI_INT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
523 if (MPI_Pack(&(gm->mode), 1, MPI_INT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
524 if (MPI_Pack(&(gm->L), 1, MPI_INT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
525 if (MPI_Pack(gm->tsc, p7P_NTRANS *M, MPI_FLOAT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
526 if (MPI_Pack(gm->rsc[0], (M+1)* Kp * p7P_NR, MPI_FLOAT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
527 if (MPI_Pack(gm->xsc[0], p7P_NXTRANS, MPI_FLOAT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
528 if (MPI_Pack(gm->xsc[1], p7P_NXTRANS, MPI_FLOAT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
529 if (MPI_Pack(gm->xsc[2], p7P_NXTRANS, MPI_FLOAT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
530 if (MPI_Pack(gm->xsc[3], p7P_NXTRANS, MPI_FLOAT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
531 if (MPI_Pack(&(gm->nj), 1, MPI_FLOAT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
532 if ((status = esl_mpi_PackOpt(gm->name, -1, MPI_CHAR, *buf, n, &position, comm)) != eslOK) goto ERROR;
533 if ((status = esl_mpi_PackOpt(gm->acc, -1, MPI_CHAR, *buf, n, &position, comm)) != eslOK) goto ERROR;
534 if ((status = esl_mpi_PackOpt(gm->desc, -1, MPI_CHAR, *buf, n, &position, comm)) != eslOK) goto ERROR;
535 if (MPI_Pack(gm->rf, M+2, MPI_CHAR, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
536 if (MPI_Pack(gm->mm, M+2, MPI_CHAR, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
537 if (MPI_Pack(gm->cs, M+2, MPI_CHAR, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
538 if (MPI_Pack(gm->consensus, M+2, MPI_CHAR, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
539 if (MPI_Pack(gm->evparam, p7_NEVPARAM, MPI_FLOAT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
540 if (MPI_Pack(gm->cutoff, p7_NCUTOFFS, MPI_FLOAT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
541 if (MPI_Pack(gm->compo, p7_MAXABET, MPI_FLOAT, *buf, n, &position, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
542 }
543
544 /* Send the packed profile to destination */
545 MPI_Send(*buf, n, MPI_PACKED, dest, tag, comm);
546 return eslOK;
547
548 ERROR:
549 return status;
550 }
551
552
553 /* Function: p7_profile_MPIRecv()
554 * Synopsis: Receive a profile as an MPI message.
555 *
556 * Purpose: Receive a profile from <source> (where <source> is usually
557 * process 0, the master) with tag <tag> from communicator <comm>,
558 * and return it in <*ret_gm>.
559 *
560 * Caller must also provide the alphabet <abc> and the
561 * background model <bg> for this profile. (Of course, that means
562 * the caller already knows them, by an appropriate
563 * initialization.)
564 *
565 * To minimize alloc/free cycles in this routine, caller
566 * passes a pointer to a buffer <*buf> of size <*nalloc>
567 * characters. These are passed by reference because if
568 * necessary, <buf> will be reallocated and <nalloc>
569 * increased to the new size. As a special case, if <buf>
570 * is <NULL> and <nalloc> is 0, the buffer will be
571 * allocated appropriately, but the caller is still
572 * responsible for free'ing it.
573 *
574 * If the packed profile is an end-of-data signal, return
575 * <eslEOD>, and <*ret_gm> is <NULL>.
576 *
577 * Returns: <eslOK> on success. <*ret_gm> contains the new profile; it
578 * is allocated here, and the caller is responsible for
579 * free'ing it. <*buf> may have been reallocated to a
580 * larger size, and <*nalloc> may have been increased.
581 *
582 */
583 int
p7_profile_MPIRecv(int source,int tag,MPI_Comm comm,const ESL_ALPHABET * abc,const P7_BG * bg,char ** buf,int * nalloc,P7_PROFILE ** ret_gm)584 p7_profile_MPIRecv(int source, int tag, MPI_Comm comm, const ESL_ALPHABET *abc, const P7_BG *bg, char **buf, int *nalloc, P7_PROFILE **ret_gm)
585 {
586 int status;
587 P7_PROFILE *gm = NULL;
588 int n;
589 int position;
590 MPI_Status mpistatus;
591 int M;
592
593 /* Probe first, because we need to know if our buffer is big enough.
594 */
595 MPI_Probe(source, tag, comm, &mpistatus);
596 MPI_Get_count(&mpistatus, MPI_PACKED, &n);
597
598 /* Make sure the buffer is allocated appropriately */
599 if (*buf == NULL || n > *nalloc) {
600 void *tmp;
601 ESL_RALLOC(*buf, tmp, sizeof(char) * n);
602 *nalloc = n;
603 }
604
605 /* Receive the packed profile */
606 MPI_Recv(*buf, n, MPI_PACKED, source, tag, comm, &mpistatus);
607
608 /* Unpack it - watching out for the EOD signal of M = -1. */
609 position = 0;
610 if (MPI_Unpack(*buf, n, &position, &M, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
611 if (M == -1) { *ret_gm = NULL; return eslEOD; }
612
613 if ((gm = p7_profile_Create(M, abc)) == NULL) { status = eslEMEM; goto ERROR; }
614 if (MPI_Unpack(*buf, n, &position, &(gm->mode), 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
615 if (MPI_Unpack(*buf, n, &position, &(gm->L), 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
616 if (MPI_Unpack(*buf, n, &position, gm->tsc, p7P_NTRANS*M, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
617 if (MPI_Unpack(*buf, n, &position, gm->rsc[0], p7P_NR*(M+1)*abc->Kp, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
618 if (MPI_Unpack(*buf, n, &position, gm->xsc[0], p7P_NXTRANS, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
619 if (MPI_Unpack(*buf, n, &position, gm->xsc[1], p7P_NXTRANS, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
620 if (MPI_Unpack(*buf, n, &position, gm->xsc[2], p7P_NXTRANS, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
621 if (MPI_Unpack(*buf, n, &position, gm->xsc[3], p7P_NXTRANS, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
622 if (MPI_Unpack(*buf, n, &position, &(gm->nj), 1, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
623
624 if ((status = esl_mpi_UnpackOpt( *buf, n, &position, (void**)&(gm->name), NULL, MPI_CHAR, comm)) != eslOK) goto ERROR;
625 if ((status = esl_mpi_UnpackOpt( *buf, n, &position, (void**)&(gm->acc), NULL, MPI_CHAR, comm)) != eslOK) goto ERROR;
626 if ((status = esl_mpi_UnpackOpt( *buf, n, &position, (void**)&(gm->desc), NULL, MPI_CHAR, comm)) != eslOK) goto ERROR;
627
628 if (MPI_Unpack(*buf, n, &position, gm->rf, M+2, MPI_CHAR, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
629 if (MPI_Unpack(*buf, n, &position, gm->mm, M+2, MPI_CHAR, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
630 if (MPI_Unpack(*buf, n, &position, gm->cs, M+2, MPI_CHAR, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
631 if (MPI_Unpack(*buf, n, &position, gm->consensus, M+2, MPI_CHAR, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
632 if (MPI_Unpack(*buf, n, &position, gm->evparam, p7_NEVPARAM, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
633 if (MPI_Unpack(*buf, n, &position, gm->cutoff, p7_NCUTOFFS, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
634 if (MPI_Unpack(*buf, n, &position, gm->compo, p7_NCUTOFFS, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
635
636 gm->abc = abc;
637 gm->M = M;
638 *ret_gm = gm;
639 return eslOK;
640
641 ERROR:
642 if (gm != NULL) p7_profile_Destroy(gm);
643 *ret_gm = NULL;
644 return status;
645 }
646
647 /*--------------- end, P7_PROFILE communication -----------------*/
648
649
650 /*****************************************************************
651 * 3. Communicating P7_PIPELINE
652 *****************************************************************/
653
654 /* Function: p7_pipeline_MPISend()
655 * Synopsis: Send pipeline data as an MPI message.
656 *
657 * Purpose: Sends pipeline statistics <pli> to MPI process <dest>
658 * (where <dest> ranges from 0..<nproc-1>), with MPI tag
659 * <tag> for MPI communicator <comm>.
660 *
661 * In order to minimize alloc/free cycles in this routine,
662 * caller passes a pointer to a working buffer <*buf> of
663 * size <*nalloc> characters. If necessary (i.e. if <pli> is
664 * too big to fit), <*buf> will be reallocated and <*nalloc>
665 * increased to the new size. As a special case, if <*buf>
666 * is <NULL> and <*nalloc> is 0, the buffer will be
667 * allocated appropriately, but the caller is still
668 * responsible for free'ing it.
669 *
670 * If <pli> is NULL, the pipeline statistics are initialized
671 * to zeros.
672 *
673 * Returns: <eslOK> on success.
674 */
675 int
p7_pipeline_MPISend(P7_PIPELINE * pli,int dest,int tag,MPI_Comm comm,char ** buf,int * nalloc)676 p7_pipeline_MPISend(P7_PIPELINE *pli, int dest, int tag, MPI_Comm comm, char **buf, int *nalloc)
677 {
678 int status;
679 int sz, n, pos;
680
681 P7_PIPELINE bogus;
682
683 /* This will look wasteful, but the MPI spec doesn't guarantee that
684 * MPI_Pack_size(x, ...) + MPI_Pack_size(x, ... ) == MPI_Pack_size(2x, ...).
685 * Indeed there are some hints in the spec that that's *not* true.
686 * So we assume we must match our Pack_size calls exactly to our Pack calls.
687 */
688 n = 0;
689 if (MPI_Pack_size(1, MPI_LONG_INT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz;
690 if (MPI_Pack_size(1, MPI_LONG_INT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz;
691 if (MPI_Pack_size(1, MPI_UINT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz;
692 if (MPI_Pack_size(1, MPI_UINT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz;
693 if (MPI_Pack_size(1, MPI_UINT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz;
694 if (MPI_Pack_size(1, MPI_UINT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz;
695 if (MPI_Pack_size(1, MPI_UINT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz;
696 if (MPI_Pack_size(1, MPI_UINT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz;
697 if (MPI_Pack_size(1, MPI_UINT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz;
698 if (MPI_Pack_size(1, MPI_UINT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz;
699 if (MPI_Pack_size(1, MPI_DOUBLE, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz;
700
701 /* Make sure the buffer is allocated appropriately */
702 if (*buf == NULL || n > *nalloc) {
703 void *tmp;
704 ESL_RALLOC(*buf, tmp, sizeof(char) * n);
705 *nalloc = n;
706 }
707
708 /* if no pipeline was defined, return zeros for the stats */
709 if (pli == NULL)
710 {
711 bogus.mode = p7_SEARCH_SEQS; /* that's 0. (some compilers complain if you set 0 directly. */
712 bogus.Z_setby = p7_ZSETBY_NTARGETS; /* ditto. */
713 bogus.nmodels = 0;
714 bogus.nseqs = 0;
715 bogus.nres = 0;
716 bogus.nnodes = 0;
717 bogus.n_past_msv = 0;
718 bogus.n_past_bias = 0;
719 bogus.n_past_vit = 0;
720 bogus.n_past_fwd = 0;
721 bogus.Z = 0.0;
722 pli = &bogus;
723 }
724
725 /* Pack the pipeline into the buffer */
726 pos = 0;
727 if (MPI_Pack(&pli->mode, 1, MPI_LONG_INT, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
728 if (MPI_Pack(&pli->Z_setby, 1, MPI_LONG_INT, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
729 if (MPI_Pack(&pli->nmodels, 1, MPI_UINT64_T, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
730 if (MPI_Pack(&pli->nseqs, 1, MPI_UINT64_T, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
731 if (MPI_Pack(&pli->nres, 1, MPI_UINT64_T, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
732 if (MPI_Pack(&pli->nnodes, 1, MPI_UINT64_T, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
733 if (MPI_Pack(&pli->n_past_msv, 1, MPI_UINT64_T, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
734 if (MPI_Pack(&pli->n_past_bias, 1, MPI_UINT64_T, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
735 if (MPI_Pack(&pli->n_past_vit, 1, MPI_UINT64_T, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
736 if (MPI_Pack(&pli->n_past_fwd, 1, MPI_UINT64_T, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
737 if (MPI_Pack(&pli->Z, 1, MPI_DOUBLE, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
738
739 /* Send the packed pipeline to destination */
740 MPI_Send(*buf, n, MPI_PACKED, dest, tag, comm);
741 return eslOK;
742
743 ERROR:
744 return status;
745 }
746
747
748 /* Function: p7_pipeline_MPIRecv()
749 * Synopsis: Receive pipeline data as an MPI message.
750 *
751 * Purpose: Receive a pipeline from <source> (where <source> is usually
752 * process 0, the master) with tag <tag> from communicator <comm>,
753 * and return it in <*ret_pli>.
754 *
755 * To minimize alloc/free cycles in this routine, caller
756 * passes a pointer to a buffer <*buf> of size <*nalloc>
757 * characters. These are passed by reference because if
758 * necessary, <buf> will be reallocated and <nalloc>
759 * increased to the new size. As a special case, if <buf>
760 * is <NULL> and <nalloc> is 0, the buffer will be
761 * allocated appropriately, but the caller is still
762 * responsible for free'ing it.
763 *
764 * Returns: <eslOK> on success. <*ret_pli> contains the new pipeline;
765 * it is allocated here, and the caller is responsible for
766 * free'ing it. <*buf> may have been reallocated to a
767 * larger size, and <*nalloc> may have been increased.
768 *
769 */
770 int
p7_pipeline_MPIRecv(int source,int tag,MPI_Comm comm,char ** buf,int * nalloc,ESL_GETOPTS * go,P7_PIPELINE ** ret_pli)771 p7_pipeline_MPIRecv(int source, int tag, MPI_Comm comm, char **buf, int *nalloc, ESL_GETOPTS *go, P7_PIPELINE **ret_pli)
772 {
773 int status;
774 P7_PIPELINE *pli = NULL;
775 int n;
776 int pos;
777 MPI_Status mpistatus;
778
779 /* Probe first, because we need to know if our buffer is big enough.
780 */
781 MPI_Probe(source, tag, comm, &mpistatus);
782 MPI_Get_count(&mpistatus, MPI_PACKED, &n);
783
784 /* Make sure the buffer is allocated appropriately */
785 if (*buf == NULL || n > *nalloc) {
786 void *tmp;
787 ESL_RALLOC(*buf, tmp, sizeof(char) * n);
788 *nalloc = n;
789 }
790
791 /* Receive the packed pipeline */
792 MPI_Recv(*buf, n, MPI_PACKED, source, tag, comm, &mpistatus);
793
794 /* Unpack it - watching out for the EOD signal of M = -1. */
795 pos = 0;
796 if ((pli = p7_pipeline_Create(go, 0, 0, FALSE, p7_SEARCH_SEQS)) == NULL) { status = eslEMEM; goto ERROR; } /* mode will be immediately overwritten */
797 if (MPI_Unpack(*buf, n, &pos, &(pli->mode), 1, MPI_LONG_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
798 if (MPI_Unpack(*buf, n, &pos, &(pli->Z_setby), 1, MPI_LONG_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
799 if (MPI_Unpack(*buf, n, &pos, &(pli->nmodels), 1, MPI_UINT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
800 if (MPI_Unpack(*buf, n, &pos, &(pli->nseqs), 1, MPI_UINT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
801 if (MPI_Unpack(*buf, n, &pos, &(pli->nres), 1, MPI_UINT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
802 if (MPI_Unpack(*buf, n, &pos, &(pli->nnodes), 1, MPI_UINT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
803 if (MPI_Unpack(*buf, n, &pos, &(pli->n_past_msv), 1, MPI_UINT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
804 if (MPI_Unpack(*buf, n, &pos, &(pli->n_past_bias), 1, MPI_UINT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
805 if (MPI_Unpack(*buf, n, &pos, &(pli->n_past_vit), 1, MPI_UINT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
806 if (MPI_Unpack(*buf, n, &pos, &(pli->n_past_fwd), 1, MPI_UINT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
807 if (MPI_Unpack(*buf, n, &pos, &(pli->Z), 1, MPI_DOUBLE, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
808
809 *ret_pli = pli;
810 return eslOK;
811
812 ERROR:
813 if (pli != NULL) p7_pipeline_Destroy(pli);
814 *ret_pli = NULL;
815 return status;
816 }
817
818 /*--------------- end, P7_PIPELINE communication -----------------*/
819
820
821 /*****************************************************************
822 * 4. Communicating P7_TOPHITS
823 *****************************************************************/
824
825 /* Function: p7_tophits_MPISend()
826 * Synopsis: Send the TOPHITS as an MPI work unit.
827 *
828 * Purpose: Sends the TOPHITS <th> as a work unit to MPI process
829 * <dest> (where <dest> ranges from 0..<nproc-1>), tagged
830 * with MPI tag <tag>, for MPI communicator <comm>, as
831 * the sole workunit or result.
832 *
833 * After the TOPHITS <th> information has been sent, send
834 * the each hit as an indepentant message.
835 *
836 * Returns: <eslOK> on success; <*buf> may have been reallocated and
837 * <*nalloc> may have been increased.
838 *
839 * Throws: <eslESYS> if an MPI call fails; <eslEMEM> if a malloc/realloc
840 * fails. In either case, <*buf> and <*nalloc> remain valid and useful
841 * memory (though the contents of <*buf> are undefined).
842 */
843 int
p7_tophits_MPISend(P7_TOPHITS * th,int dest,int tag,MPI_Comm comm,char ** buf,int * nalloc)844 p7_tophits_MPISend(P7_TOPHITS *th, int dest, int tag, MPI_Comm comm, char **buf, int *nalloc)
845 {
846 int status;
847 int sz, n, pos;
848 int i, j, inx;
849
850 P7_DOMAIN *dcl = NULL;
851 P7_HIT *hit = NULL;
852
853 n = 0;
854 sz = 0;
855
856 /* calculate the buffer size needed to hold the largest hit */
857 hit = th->unsrt;
858 for (i = 0; i < th->N; ++i) {
859 for (j = 0; j < th->unsrt[i].ndom; ++j) {
860 if (sz <= hit->dcl[j].ad->memsize) {
861 sz = hit->dcl[j].ad->memsize;
862 dcl = &hit->dcl[j];
863 }
864 }
865 ++hit;
866 }
867
868 if (th->N > 0) {
869 if ((status = p7_hit_MPIPackSize(th->unsrt, comm, &n)) != eslOK) goto ERROR;
870 if (dcl != NULL) {
871 if ((status = p7_dcl_MPIPackSize(dcl, comm, &sz)) != eslOK) goto ERROR;
872 n = (n > sz) ? n : sz;
873 }
874 }
875
876 if (MPI_Pack_size(3, MPI_UINT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed");
877 n = (n > sz) ? n : sz;
878
879 /* Make sure the buffer is allocated appropriately */
880 if (*buf == NULL || n > *nalloc) {
881 void *tmp;
882 ESL_RALLOC(*buf, tmp, sizeof(char) * n);
883 *nalloc = n;
884 }
885
886 pos = 0;
887 if (MPI_Pack(&th->N, 1, MPI_UINT64_T, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
888 if (MPI_Pack(&th->nreported, 1, MPI_UINT64_T, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
889 if (MPI_Pack(&th->nincluded, 1, MPI_UINT64_T, *buf, n, &pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
890
891 /* Send the packed tophits information */
892 if (MPI_Send(*buf, n, MPI_PACKED, dest, tag, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi send failed");
893 if (th->N == 0) return eslOK;
894
895 /* loop through the hit list sending to dest */
896 hit = th->unsrt;
897 for (inx = 0; inx < th->N; ++inx) {
898 if ((status = p7_hit_MPISend(hit, dest, tag, comm, buf, nalloc)) != eslOK) goto ERROR;
899 ++hit;
900 }
901
902 return eslOK;
903
904 ERROR:
905 return status;
906 }
907
908 /* Function: p7_tophits_MPIRecv()
909 * Synopsis: Receives an TOPHITS as a work unit from an MPI sender.
910 *
911 * Purpose: Sends the TOPHITS <th> as a work unit to MPI process
912 * <dest> (where <dest> ranges from 0..<nproc-1>), tagged
913 * with MPI tag <tag>, for MPI communicator <comm>, as
914 * the sole workunit or result.
915 *
916 * After the TOPHITS <th> information has been sent, send
917 * the each hit as an indepentant message.
918 *
919 * Returns: <eslOK> on success; <*buf> may have been reallocated and
920 * <*nalloc> may have been increased.
921 *
922 * Throws: <eslESYS> if an MPI call fails; <eslEMEM> if a malloc/realloc
923 * fails. In either case, <*buf> and <*nalloc> remain valid and useful
924 * memory (though the contents of <*buf> are undefined).
925 */
926 int
p7_tophits_MPIRecv(int source,int tag,MPI_Comm comm,char ** buf,int * nalloc,P7_TOPHITS ** ret_th)927 p7_tophits_MPIRecv(int source, int tag, MPI_Comm comm, char **buf, int *nalloc, P7_TOPHITS **ret_th)
928 {
929 int n;
930 int status;
931 int pos;
932 P7_TOPHITS *th = NULL;
933 P7_HIT *hit = NULL;
934 MPI_Status mpistatus;
935
936 uint64_t nhits;
937 uint64_t inx;
938
939 /* Probe first, because we need to know if our buffer is big enough.
940 */
941 MPI_Probe(source, tag, comm, &mpistatus);
942 MPI_Get_count(&mpistatus, MPI_PACKED, &n);
943
944 /* make sure we are getting the tag we expect and from whom we expect if from */
945 if (tag != MPI_ANY_TAG && mpistatus.MPI_TAG != tag) {
946 status = eslFAIL;
947 goto ERROR;
948 }
949 if (source != MPI_ANY_SOURCE && mpistatus.MPI_SOURCE != source) {
950 status = eslFAIL;
951 goto ERROR;
952 }
953
954 /* set the source and tag */
955 tag = mpistatus.MPI_TAG;
956 source = mpistatus.MPI_SOURCE;
957
958 /* Make sure the buffer is allocated appropriately */
959 if (*buf == NULL || n > *nalloc) {
960 void *tmp;
961 ESL_RALLOC(*buf, tmp, sizeof(char) * n);
962 *nalloc = n;
963 }
964
965 /* Receive the packed top hits */
966 MPI_Recv(*buf, n, MPI_PACKED, source, tag, comm, &mpistatus);
967
968 /* Unpack it - watching out for the EOD signal of M = -1. */
969 pos = 0;
970 if ((th = p7_tophits_Create()) == NULL) { status = eslEMEM; goto ERROR; }
971 if (MPI_Unpack(*buf, n, &pos, &nhits, 1, MPI_UINT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
972 if (MPI_Unpack(*buf, n, &pos, &th->nreported, 1, MPI_UINT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
973 if (MPI_Unpack(*buf, n, &pos, &th->nincluded, 1, MPI_UINT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "unpack failed");
974
975 /* loop through all of the hits sent */
976 for (inx = 0; inx < nhits; ++inx) {
977 if ((status = p7_tophits_CreateNextHit(th, &hit)) != eslOK) goto ERROR;
978 if ((status = p7_hit_MPIRecv(source, tag, comm, buf, nalloc, hit)) != eslOK) goto ERROR;
979 }
980 *ret_th = th;
981 return eslOK;
982
983 ERROR:
984 if (th != NULL) p7_tophits_Destroy(th);
985 *ret_th = NULL;
986 return status;
987 }
988
989
990 /* Function: p7_hit_MPISend()
991 */
992 int
p7_hit_MPISend(P7_HIT * hit,int dest,int tag,MPI_Comm comm,char ** buf,int * nalloc)993 p7_hit_MPISend(P7_HIT *hit, int dest, int tag, MPI_Comm comm, char **buf, int *nalloc)
994 {
995 int inx;
996 int status;
997 int pos;
998 int n = *nalloc;
999
1000 /* Pack the HIT into the buffer */
1001 pos = 0;
1002 if ((status = p7_hit_MPIPack(hit, *buf, n, &pos, comm)) != eslOK) goto ERROR;
1003
1004 /* Send the packed HIT to the destination. */
1005 if (MPI_Send(*buf, n, MPI_PACKED, dest, tag, comm) != 0) ESL_EXCEPTION(eslESYS, "mpi send failed");
1006
1007 /* loop through all of the domains */
1008 for (inx = 0; inx < hit->ndom; ++inx) {
1009 if ((status = p7_dcl_MPISend(hit->dcl + inx, dest, tag, comm, buf, nalloc)) != eslOK) goto ERROR;
1010 }
1011
1012 return eslOK;
1013
1014 ERROR:
1015 return status;
1016 }
1017
1018 /* Function: p7_hit_MPIRecv()
1019 */
1020 int
p7_hit_MPIRecv(int source,int tag,MPI_Comm comm,char ** buf,int * nalloc,P7_HIT * hit)1021 p7_hit_MPIRecv(int source, int tag, MPI_Comm comm, char **buf, int *nalloc, P7_HIT *hit)
1022 {
1023 int n;
1024 int status;
1025 int pos;
1026 int inx;
1027 MPI_Status mpistatus;
1028
1029 /* Probe first, because we need to know if our buffer is big enough.
1030 */
1031 MPI_Probe(source, tag, comm, &mpistatus);
1032 MPI_Get_count(&mpistatus, MPI_PACKED, &n);
1033
1034 /* make sure we are getting the tag we expect and from whom we expect if from */
1035 if (tag != MPI_ANY_TAG && mpistatus.MPI_TAG != tag) {
1036 status = eslFAIL;
1037 goto ERROR;
1038 }
1039 if (source != MPI_ANY_SOURCE && mpistatus.MPI_SOURCE != source) {
1040 status = eslFAIL;
1041 goto ERROR;
1042 }
1043
1044 /* set the source and tag */
1045 tag = mpistatus.MPI_TAG;
1046 source = mpistatus.MPI_SOURCE;
1047
1048 /* Make sure the buffer is allocated appropriately */
1049 if (*buf == NULL || n > *nalloc) {
1050 void *tmp;
1051 ESL_RALLOC(*buf, tmp, sizeof(char) * n);
1052 *nalloc = n;
1053 }
1054
1055 /* Receive the packed top hits */
1056 MPI_Recv(*buf, n, MPI_PACKED, source, tag, comm, &mpistatus);
1057
1058 /* Unpack it - watching out for the EOD signal of M = -1. */
1059 pos = 0;
1060 if ((status = p7_hit_MPIUnpack(*buf, n, &pos, comm, hit)) != eslOK) goto ERROR;
1061 ESL_ALLOC(hit->dcl, sizeof(P7_DOMAIN) * hit->ndom);
1062
1063 /* loop through all of the hits sent */
1064 for (inx = 0; inx < hit->ndom; ++inx) {
1065 if ((status = p7_dcl_MPIRecv(source, tag, comm, buf, nalloc, hit->dcl + inx)) != eslOK) goto ERROR;
1066 }
1067 return eslOK;
1068
1069 ERROR:
1070 return status;
1071 }
1072
1073 /* Function: p7_hit_MPIPackSize()
1074 * Synopsis: Calculates size needed to pack a HIT.
1075 *
1076 * Purpose: Calculate an upper bound on the number of bytes
1077 * that <p7_hit_MPIPack()> will need to pack an P7_HIT
1078 * <hit> in a packed MPI message for MPI communicator
1079 * <comm>; return that number of bytes in <*ret_n>.
1080 *
1081 * Returns: <eslOK> on success, and <*ret_n> contains the answer.
1082 *
1083 * Throws: <eslESYS> if an MPI call fails, and <*ret_n> is 0.
1084 */
1085 int
p7_hit_MPIPackSize(P7_HIT * hit,MPI_Comm comm,int * ret_n)1086 p7_hit_MPIPackSize(P7_HIT *hit, MPI_Comm comm, int *ret_n)
1087 {
1088 int status;
1089 int n = 0;
1090 int sz;
1091
1092 /* P7_HIT data */
1093 if (MPI_Pack_size(1, MPI_INT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* window_length */
1094 if (MPI_Pack_size(1, MPI_DOUBLE, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* sortkey */
1095 if (MPI_Pack_size(3, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* scores */
1096 if (MPI_Pack_size(3, MPI_DOUBLE, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* lnP */
1097 if (MPI_Pack_size(1, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* expected */
1098 if (MPI_Pack_size(5, MPI_INT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* region, envelopes, ndom */
1099 if (MPI_Pack_size(3, MPI_INT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* report info */
1100 if (MPI_Pack_size(1, MPI_UINT32_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* flags */
1101 if (MPI_Pack_size(2, MPI_INT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* seqidx, subseq_start */
1102 if ((status = esl_mpi_PackOptSize(hit->name, -1, MPI_CHAR, comm, &sz)) != eslOK) goto ERROR; else n += sz;
1103 if ((status = esl_mpi_PackOptSize(hit->acc, -1, MPI_CHAR, comm, &sz)) != eslOK) goto ERROR; else n += sz;
1104 if ((status = esl_mpi_PackOptSize(hit->desc, -1, MPI_CHAR, comm, &sz)) != eslOK) goto ERROR; else n += sz;
1105
1106 *ret_n = n;
1107 return eslOK;
1108
1109 ERROR:
1110 *ret_n = 0;
1111 return status;
1112
1113 }
1114
1115 /* Function: p7_hit_MPIPack()
1116 * Synopsis: Packs the HIT into MPI buffer.
1117 *
1118 * Purpose: Packs HIT <hit> into an MPI packed message buffer <buf>
1119 * of length <n> bytes, starting at byte position <*position>,
1120 * for MPI communicator <comm>.
1121 *
1122 * The caller must know that <buf>'s allocation of <n>
1123 * bytes is large enough to append the packed HIT at
1124 * position <*pos>. This typically requires a call to
1125 * <p7_hit_MPIPackSize()> first, and reallocation if
1126 * needed.
1127 *
1128 * Returns: <eslOK> on success; <buf> now contains the
1129 * packed <hit>, and <*position> is set to the byte
1130 * immediately following the last byte of the HIT
1131 * in <buf>.
1132 *
1133 * Throws: <eslESYS> if an MPI call fails; or <eslEMEM> if the
1134 * buffer's length <n> was overflowed in trying to pack
1135 * <msa> into <buf>. In either case, the state of
1136 * <buf> and <*position> is undefined, and both should
1137 * be considered to be corrupted.
1138 */
1139 int
p7_hit_MPIPack(P7_HIT * hit,char * buf,int n,int * pos,MPI_Comm comm)1140 p7_hit_MPIPack(P7_HIT *hit, char *buf, int n, int *pos, MPI_Comm comm)
1141 {
1142 int status;
1143 if (MPI_Pack(&hit->window_length, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1144 if (MPI_Pack(&hit->sortkey, 1, MPI_DOUBLE, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1145 if (MPI_Pack(&hit->score, 1, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1146 if (MPI_Pack(&hit->pre_score, 1, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1147 if (MPI_Pack(&hit->sum_score, 1, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1148 if (MPI_Pack(&hit->lnP, 1, MPI_DOUBLE, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1149 if (MPI_Pack(&hit->pre_lnP, 1, MPI_DOUBLE, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1150 if (MPI_Pack(&hit->sum_lnP, 1, MPI_DOUBLE, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1151 if (MPI_Pack(&hit->nexpected, 1, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1152 if (MPI_Pack(&hit->nregions, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1153 if (MPI_Pack(&hit->nclustered, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1154 if (MPI_Pack(&hit->noverlaps, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1155 if (MPI_Pack(&hit->nenvelopes, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1156 if (MPI_Pack(&hit->ndom, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1157 if (MPI_Pack(&hit->flags, 1, MPI_UINT32_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1158 if (MPI_Pack(&hit->nreported, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1159 if (MPI_Pack(&hit->nincluded, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1160 if (MPI_Pack(&hit->best_domain, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1161 if (MPI_Pack(&hit->seqidx, 1, MPI_INT64_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1162 if (MPI_Pack(&hit->subseq_start, 1, MPI_INT64_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1163
1164 if ((status = esl_mpi_PackOpt(hit->name, -1, MPI_CHAR, buf, n, pos, comm)) != eslOK) return status;
1165 if ((status = esl_mpi_PackOpt(hit->acc, -1, MPI_CHAR, buf, n, pos, comm)) != eslOK) return status;
1166 if ((status = esl_mpi_PackOpt(hit->desc, -1, MPI_CHAR, buf, n, pos, comm)) != eslOK) return status;
1167
1168 if (*pos > n) ESL_EXCEPTION(eslEMEM, "buffer overflow");
1169 return eslOK;
1170
1171 ERROR:
1172 return status;
1173 }
1174
1175 /* Function: p7_hit_MPIUnpack()
1176 * Synopsis: Unpacks an HIT from an MPI buffer.
1177 *
1178 * Purpose: Unpack a newly allocated HIT from MPI packed buffer
1179 * <buf>, starting from position <*pos>, where the total length
1180 * of the buffer in bytes is <n>.
1181 *
1182 * Returns: <eslOK> on success. <*pos> is updated to the position of
1183 * the next element in <buf> to unpack (if any). <*ret_hit>
1184 * contains a newly allocated HIT, which the caller is
1185 * responsible for free'ing.
1186 *
1187 * Throws: <eslESYS> on an MPI call failure. <eslEMEM> on allocation failure.
1188 * In either case, <*ret_hit> is <NULL>, and the state of <buf>
1189 * and <*pos> is undefined and should be considered to be corrupted.
1190 */
1191 int
p7_hit_MPIUnpack(char * buf,int n,int * pos,MPI_Comm comm,P7_HIT * hit)1192 p7_hit_MPIUnpack(char *buf, int n, int *pos, MPI_Comm comm, P7_HIT *hit)
1193 {
1194 int status;
1195
1196 if (MPI_Unpack(buf, n, pos, &hit->window_length, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1197 if (MPI_Unpack(buf, n, pos, &hit->sortkey, 1, MPI_DOUBLE, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1198 if (MPI_Unpack(buf, n, pos, &hit->score, 1, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1199 if (MPI_Unpack(buf, n, pos, &hit->pre_score, 1, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1200 if (MPI_Unpack(buf, n, pos, &hit->sum_score, 1, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1201 if (MPI_Unpack(buf, n, pos, &hit->lnP, 1, MPI_DOUBLE, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1202 if (MPI_Unpack(buf, n, pos, &hit->pre_lnP, 1, MPI_DOUBLE, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1203 if (MPI_Unpack(buf, n, pos, &hit->sum_lnP, 1, MPI_DOUBLE, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1204 if (MPI_Unpack(buf, n, pos, &hit->nexpected, 1, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1205 if (MPI_Unpack(buf, n, pos, &hit->nregions, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1206 if (MPI_Unpack(buf, n, pos, &hit->nclustered, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1207 if (MPI_Unpack(buf, n, pos, &hit->noverlaps, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1208 if (MPI_Unpack(buf, n, pos, &hit->nenvelopes, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1209 if (MPI_Unpack(buf, n, pos, &hit->ndom, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1210 if (MPI_Unpack(buf, n, pos, &hit->flags, 1, MPI_UINT32_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1211 if (MPI_Unpack(buf, n, pos, &hit->nreported, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1212 if (MPI_Unpack(buf, n, pos, &hit->nincluded, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1213 if (MPI_Unpack(buf, n, pos, &hit->best_domain, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1214 if (MPI_Unpack(buf, n, pos, &hit->seqidx, 1, MPI_INT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1215 if (MPI_Unpack(buf, n, pos, &hit->subseq_start, 1, MPI_INT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1216 hit->offset = 0; // This field is only used when packing search results for transmission over sockets (not MPI)
1217 // and its value isn't guaranteed to be the same on different machines, so just set it to 0
1218
1219 if ((status = esl_mpi_UnpackOpt(buf, n, pos, (void**)&(hit->name), NULL, MPI_CHAR, comm)) != eslOK) goto ERROR;
1220 if ((status = esl_mpi_UnpackOpt(buf, n, pos, (void**)&(hit->acc), NULL, MPI_CHAR, comm)) != eslOK) goto ERROR;
1221 if ((status = esl_mpi_UnpackOpt(buf, n, pos, (void**)&(hit->desc), NULL, MPI_CHAR, comm)) != eslOK) goto ERROR;
1222
1223 return eslOK;
1224
1225 ERROR:
1226 return status;
1227 }
1228
1229 /* Function: p7_dcl_MPISend()
1230 */
1231 int
p7_dcl_MPISend(P7_DOMAIN * dcl,int dest,int tag,MPI_Comm comm,char ** buf,int * nalloc)1232 p7_dcl_MPISend(P7_DOMAIN *dcl, int dest, int tag, MPI_Comm comm, char **buf, int *nalloc)
1233 {
1234 int status;
1235 int pos;
1236 int n = *nalloc;
1237
1238 /* Pack the DOMAIN into the buffer */
1239 pos = 0;
1240 if ((status = p7_dcl_MPIPack(dcl, *buf, n, &pos, comm)) != eslOK) goto ERROR;
1241
1242 /* Send the packed HIT to the destination. */
1243 if (MPI_Send(*buf, n, MPI_PACKED, dest, tag, comm) != 0) ESL_EXCEPTION(eslESYS, "mpi send failed");
1244
1245 return eslOK;
1246
1247 ERROR:
1248 return status;
1249 }
1250
1251
1252 /* Function: p7_dcl_MPIPackSize()
1253 */
1254 int
p7_dcl_MPIPackSize(P7_DOMAIN * dcl,MPI_Comm comm,int * ret_n)1255 p7_dcl_MPIPackSize(P7_DOMAIN *dcl, MPI_Comm comm, int *ret_n)
1256 {
1257 int status;
1258 int n = 0;
1259 int sz;
1260
1261 P7_ALIDISPLAY *ad = dcl->ad;
1262
1263 /* P7_DOMAIN data */
1264 if (MPI_Pack_size(6, MPI_INT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* alignment info */
1265 if (MPI_Pack_size(5, MPI_FLOAT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* score info */
1266 if (MPI_Pack_size(1, MPI_DOUBLE, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* lnP */
1267 if (MPI_Pack_size(2, MPI_INT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* thresholds */
1268 if(dcl->scores_per_pos != NULL){p7_Fail("Non-NULL scores_per_pos field found int p7_dcl_MPIPackSize. Sending the scores_per_pos field via MPI has not been implemented\n");}
1269
1270 /* P7_ALIDISPLAY data */
1271 if (MPI_Pack_size(18, MPI_INT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* offset info */
1272 if (MPI_Pack_size(3, MPI_INT64_T, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* sequence info */
1273 if (MPI_Pack_size(1, MPI_INT, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* string pool size */
1274 if (MPI_Pack_size(ad->memsize, MPI_CHAR, comm, &sz) != 0) ESL_XEXCEPTION(eslESYS, "pack size failed"); n += sz; /* string pool */
1275
1276 *ret_n = n;
1277 return eslOK;
1278
1279 ERROR:
1280 *ret_n = 0;
1281 return status;
1282
1283 }
1284
1285 /* Function: p7_dcl_MPIPack()
1286 */
1287 int
p7_dcl_MPIPack(P7_DOMAIN * dcl,char * buf,int n,int * pos,MPI_Comm comm)1288 p7_dcl_MPIPack(P7_DOMAIN *dcl, char *buf, int n, int *pos, MPI_Comm comm)
1289 {
1290 int status;
1291 int offset;
1292
1293 P7_ALIDISPLAY *ad = dcl->ad;
1294
1295 if (MPI_Pack(&dcl->ienv, 1, MPI_INT64_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1296 if (MPI_Pack(&dcl->jenv, 1, MPI_INT64_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1297 if (MPI_Pack(&dcl->iali, 1, MPI_INT64_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1298 if (MPI_Pack(&dcl->jali, 1, MPI_INT64_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1299 if (MPI_Pack(&dcl->iorf, 1, MPI_INT64_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1300 if (MPI_Pack(&dcl->jorf, 1, MPI_INT64_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1301 if (MPI_Pack(&dcl->envsc, 1, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1302 if (MPI_Pack(&dcl->domcorrection, 1, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1303 if (MPI_Pack(&dcl->dombias, 1, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1304 if (MPI_Pack(&dcl->oasc, 1, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1305 if (MPI_Pack(&dcl->bitscore, 1, MPI_FLOAT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1306 if (MPI_Pack(&dcl->lnP, 1, MPI_DOUBLE, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1307 if (MPI_Pack(&dcl->is_reported, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1308 if (MPI_Pack(&dcl->is_included, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1309 if(dcl->scores_per_pos != NULL){p7_Fail("Non-NULL scores_per_pos field found int p7_dcl_MPIPack. Sending the scores_per_pos field via MPI has not been implemented\n");}
1310
1311 offset = (ad->rfline == NULL) ? -1 : ad->rfline - ad->mem;
1312 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1313 offset = (ad->mmline == NULL) ? -1 : ad->mmline - ad->mem;
1314 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1315 offset = (ad->csline == NULL) ? -1 : ad->csline - ad->mem;
1316 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1317 offset = (ad->model == NULL) ? -1 : ad->model - ad->mem;
1318 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1319 offset = (ad->mline == NULL) ? -1 : ad->mline - ad->mem;
1320 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1321 offset = (ad->aseq == NULL) ? -1 : ad->aseq - ad->mem;
1322 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1323 offset = (ad->ntseq == NULL) ? -1 : ad->ntseq - ad->mem;
1324 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1325 offset = (ad->ppline == NULL) ? -1 : ad->ppline - ad->mem;
1326 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1327 if (MPI_Pack(&ad->N, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1328 offset = (ad->hmmname == NULL) ? -1 : ad->hmmname - ad->mem;
1329 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1330 offset = (ad->hmmacc == NULL) ? -1 : ad->hmmacc - ad->mem;
1331 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1332 offset = (ad->hmmdesc == NULL) ? -1 : ad->hmmdesc - ad->mem;
1333 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1334 if (MPI_Pack(&ad->hmmfrom, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1335 if (MPI_Pack(&ad->hmmto, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1336 if (MPI_Pack(&ad->M, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1337 offset = (ad->sqname == NULL) ? -1 : ad->sqname - ad->mem;
1338 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1339 offset = (ad->sqacc == NULL) ? -1 : ad->sqacc - ad->mem;
1340 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1341 offset = (ad->sqdesc == NULL) ? -1 : ad->sqdesc - ad->mem;
1342 if (MPI_Pack(&offset, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1343 if (MPI_Pack(&ad->sqfrom, 1, MPI_INT64_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1344 if (MPI_Pack(&ad->sqto, 1, MPI_INT64_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1345 if (MPI_Pack(&ad->L, 1, MPI_INT64_T, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1346 if (MPI_Pack(&ad->memsize, 1, MPI_INT, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1347 if (MPI_Pack( ad->mem, ad->memsize, MPI_CHAR, buf, n, pos, comm) != 0) ESL_XEXCEPTION(eslESYS, "pack failed");
1348
1349 if (*pos > n) ESL_EXCEPTION(eslEMEM, "buffer overflow");
1350 return eslOK;
1351
1352 ERROR:
1353 return status;
1354 }
1355
1356 /* Function: p7_dcl_MPIUnpack()
1357 */
1358 int
p7_dcl_MPIUnpack(char * buf,int n,int * pos,MPI_Comm comm,P7_DOMAIN * dcl)1359 p7_dcl_MPIUnpack(char *buf, int n, int *pos, MPI_Comm comm, P7_DOMAIN *dcl)
1360 {
1361 int status;
1362 int rfline, mmline, csline, model, mline, aseq, ntseq, ppline;
1363 int hmmname, hmmacc, hmmdesc;
1364 int sqname, sqacc, sqdesc;
1365
1366 P7_ALIDISPLAY *ad;
1367
1368 ESL_ALLOC(ad, sizeof(P7_ALIDISPLAY));
1369
1370 if (MPI_Unpack(buf, n, pos, &dcl->ienv, 1, MPI_INT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1371 if (MPI_Unpack(buf, n, pos, &dcl->jenv, 1, MPI_INT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1372 if (MPI_Unpack(buf, n, pos, &dcl->iali, 1, MPI_INT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1373 if (MPI_Unpack(buf, n, pos, &dcl->jali, 1, MPI_INT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1374 if (MPI_Unpack(buf, n, pos, &dcl->iorf, 1, MPI_INT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1375 if (MPI_Unpack(buf, n, pos, &dcl->jorf, 1, MPI_INT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1376 if (MPI_Unpack(buf, n, pos, &dcl->envsc, 1, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1377 if (MPI_Unpack(buf, n, pos, &dcl->domcorrection, 1, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1378 if (MPI_Unpack(buf, n, pos, &dcl->dombias, 1, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1379 if (MPI_Unpack(buf, n, pos, &dcl->oasc, 1, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1380 if (MPI_Unpack(buf, n, pos, &dcl->bitscore, 1, MPI_FLOAT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1381 if (MPI_Unpack(buf, n, pos, &dcl->lnP, 1, MPI_DOUBLE, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1382 if (MPI_Unpack(buf, n, pos, &dcl->is_reported, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1383 if (MPI_Unpack(buf, n, pos, &dcl->is_included, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1384 dcl->scores_per_pos =NULL; // we don't support sending this field over MPI, so init to NULL to avoid problems from stale memory
1385
1386 if (MPI_Unpack(buf, n, pos, &rfline, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1387 if (MPI_Unpack(buf, n, pos, &mmline, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1388 if (MPI_Unpack(buf, n, pos, &csline, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1389 if (MPI_Unpack(buf, n, pos, &model, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1390 if (MPI_Unpack(buf, n, pos, &mline, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1391 if (MPI_Unpack(buf, n, pos, &aseq, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1392 if (MPI_Unpack(buf, n, pos, &ntseq, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1393 if (MPI_Unpack(buf, n, pos, &ppline, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1394 if (MPI_Unpack(buf, n, pos, &ad->N, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1395 if (MPI_Unpack(buf, n, pos, &hmmname, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1396 if (MPI_Unpack(buf, n, pos, &hmmacc, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1397 if (MPI_Unpack(buf, n, pos, &hmmdesc, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1398 if (MPI_Unpack(buf, n, pos, &ad->hmmfrom, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1399 if (MPI_Unpack(buf, n, pos, &ad->hmmto, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1400 if (MPI_Unpack(buf, n, pos, &ad->M, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1401 if (MPI_Unpack(buf, n, pos, &sqname, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1402 if (MPI_Unpack(buf, n, pos, &sqacc, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1403 if (MPI_Unpack(buf, n, pos, &sqdesc, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1404 if (MPI_Unpack(buf, n, pos, &ad->sqfrom, 1, MPI_INT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1405 if (MPI_Unpack(buf, n, pos, &ad->sqto, 1, MPI_INT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1406 if (MPI_Unpack(buf, n, pos, &ad->L, 1, MPI_INT64_T, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1407 if (MPI_Unpack(buf, n, pos, &ad->memsize, 1, MPI_INT, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1408
1409 dcl->scores_per_pos = NULL; /*this field is used for nhmmer, which currently doesn't have MPI support */
1410
1411 /* allocate the string pools for the alignments */
1412 ESL_ALLOC(ad->mem, ad->memsize);
1413 if (MPI_Unpack(buf, n, pos, ad->mem, ad->memsize, MPI_CHAR, comm) != 0) ESL_XEXCEPTION(eslESYS, "mpi unpack failed");
1414
1415 ad->rfline = (rfline == -1) ? NULL : ad->mem + rfline;
1416 ad->mmline = (mmline == -1) ? NULL : ad->mem + mmline;
1417 ad->csline = (csline == -1) ? NULL : ad->mem + csline;
1418 ad->model = (model == -1) ? NULL : ad->mem + model;
1419 ad->mline = (mline == -1) ? NULL : ad->mem + mline;
1420 ad->aseq = (aseq == -1) ? NULL : ad->mem + aseq;
1421 ad->ntseq = (ntseq == -1) ? NULL : ad->mem + ntseq;
1422 ad->ppline = (ppline == -1) ? NULL : ad->mem + ppline;
1423
1424 ad->hmmname = (hmmname == -1) ? NULL : ad->mem + hmmname;
1425 ad->hmmacc = (hmmacc == -1) ? NULL : ad->mem + hmmacc;
1426 ad->hmmdesc = (hmmdesc == -1) ? NULL : ad->mem + hmmdesc;
1427
1428 ad->sqname = (sqname == -1) ? NULL : ad->mem + sqname;
1429 ad->sqacc = (sqacc == -1) ? NULL : ad->mem + sqacc;
1430 ad->sqdesc = (sqdesc == -1) ? NULL : ad->mem + sqdesc;
1431
1432 dcl->ad = ad;
1433
1434 return eslOK;
1435
1436 ERROR:
1437 if (ad != NULL) {
1438 if (ad->mem != NULL) free(ad->mem);
1439 free(ad);
1440 }
1441 return status;
1442 }
1443
1444 /* Function: p7_dcl_MPIRecv()
1445 */
1446 int
p7_dcl_MPIRecv(int source,int tag,MPI_Comm comm,char ** buf,int * nalloc,P7_DOMAIN * dcl)1447 p7_dcl_MPIRecv(int source, int tag, MPI_Comm comm, char **buf, int *nalloc, P7_DOMAIN *dcl)
1448 {
1449 int status;
1450 int n;
1451 int pos;
1452 MPI_Status mpistatus;
1453
1454 /* Probe first, because we need to know if our buffer is big enough.
1455 */
1456 MPI_Probe(source, tag, comm, &mpistatus);
1457 MPI_Get_count(&mpistatus, MPI_PACKED, &n);
1458
1459 /* make sure we are getting the tag we expect and from whom we expect if from */
1460 if (tag != MPI_ANY_TAG && mpistatus.MPI_TAG != tag) {
1461 status = eslFAIL;
1462 goto ERROR;
1463 }
1464 if (source != MPI_ANY_SOURCE && mpistatus.MPI_SOURCE != source) {
1465 status = eslFAIL;
1466 goto ERROR;
1467 }
1468
1469 /* set the source and tag */
1470 tag = mpistatus.MPI_TAG;
1471 source = mpistatus.MPI_SOURCE;
1472
1473 /* Make sure the buffer is allocated appropriately */
1474 if (*buf == NULL || n > *nalloc) {
1475 void *tmp;
1476 ESL_RALLOC(*buf, tmp, sizeof(char) * n);
1477 *nalloc = n;
1478 }
1479
1480 /* Receive the packed dcl */
1481 MPI_Recv(*buf, n, MPI_PACKED, source, tag, comm, &mpistatus);
1482
1483 /* Unpack it, looking at the status code prefix for EOD/EOK */
1484 pos = 0;
1485 return p7_dcl_MPIUnpack(*buf, *nalloc, &pos, comm, dcl);
1486
1487 ERROR:
1488 return status;
1489 }
1490
1491 /*----------------- end, P7_TOPHITS communication -------------------*/
1492
1493
1494 /*****************************************************************
1495 * 5. Benchmark driver.
1496 *****************************************************************/
1497
1498 #ifdef p7MPISUPPORT_BENCHMARK
1499 /*
1500 mpicc -g -Wall -L. -I. -L ../easel -I ../easel -D p7MPISUPPORT_BENCHMARK -o benchmark-mpi mpisupport.c -lhmmer -leasel -lm
1501 qsub -N benchmark-mpi -j y -R y -b y -cwd -V -pe lam-mpi-tight 2 'mpirun C ./benchmark-mpi ~/notebook/1227-msp-statistics/Pfam.hmm > bench.out'
1502 qsub -N benchmark-mpi -j y -R y -b y -cwd -V -pe lam-mpi-tight 2 'mpirun C ./benchmark-mpi -b ~/notebook/1227-msp-statistics/Pfam.hmm > bench.out'
1503 */
1504 #include "p7_config.h"
1505
1506 #include <string.h>
1507 #include <math.h>
1508
1509 #include "easel.h"
1510 #include "esl_getopts.h"
1511 #include "esl_alphabet.h"
1512 #include "esl_random.h"
1513 #include "esl_stopwatch.h"
1514
1515 #include "hmmer.h"
1516
1517 static ESL_OPTIONS options[] = {
1518 /* name type default env range toggles reqs incomp help docgroup*/
1519 { "-h", eslARG_NONE, FALSE, NULL, NULL, NULL, NULL, NULL, "show brief help on version and usage", 0 },
1520 { "-b", eslARG_NONE, FALSE, NULL, NULL, NULL, NULL, NULL, "baseline timing: don't send any HMMs", 0 },
1521 { "--stall", eslARG_NONE, FALSE, NULL, NULL, NULL, NULL, NULL, "arrest after start: for debugging MPI under gdb", 0 },
1522 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
1523 };
1524 static char usage[] = "[-options] <hmmfile>";
1525 static char banner[] = "benchmark driver for MPI communication";
1526
1527 int
main(int argc,char ** argv)1528 main(int argc, char **argv)
1529 {
1530 ESL_GETOPTS *go = p7_CreateDefaultApp(options, 1, argc, argv, banner, usage);
1531 char *hmmfile = esl_opt_GetArg(go, 1);
1532 ESL_ALPHABET *abc = esl_alphabet_Create(eslAMINO);
1533 P7_BG *bg = p7_bg_Create(abc);
1534 int my_rank;
1535 int nproc;
1536 char *buf = NULL;
1537 int nbuf = 0;
1538 int subtotalM = 0;
1539 int allM = 0;
1540 int stalling = esl_opt_GetBoolean(go, "--stall");
1541
1542 MPI_Init(&argc, &argv);
1543 MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
1544 MPI_Comm_size(MPI_COMM_WORLD, &nproc);
1545
1546 while (stalling);
1547
1548 /* Master MPI process: */
1549 if (my_rank == 0)
1550 {
1551 ESL_STOPWATCH *w = esl_stopwatch_Create();
1552 P7_HMMFILE *hfp = NULL;
1553 P7_PROFILE *gm = NULL;
1554 P7_HMM *hmm = NULL;
1555
1556
1557 /* Read HMMs from a file. */
1558 if (p7_hmmfile_OpenE(hmmfile, NULL, &hfp, NULL) != eslOK) p7_Fail("Failed to open HMM file %s", hmmfile);
1559
1560 esl_stopwatch_Start(w);
1561 while (p7_hmmfile_Read(hfp, &abc, &hmm) == eslOK)
1562 {
1563 gm = p7_profile_Create(hmm->M, abc);
1564 p7_ProfileConfig(hmm, bg, gm, 400, p7_LOCAL);
1565 if (!esl_opt_GetBoolean(go, "-b"))
1566 p7_profile_MPISend(gm, 1, 0, MPI_COMM_WORLD, &buf, &nbuf); /* 1 = dest; 0 = tag */
1567
1568 p7_hmm_Destroy(hmm);
1569 p7_profile_Destroy(gm);
1570 }
1571 p7_profile_MPISend(NULL, 1, 0, MPI_COMM_WORLD, &buf, &nbuf); /* send the "no more HMMs" sign */
1572 MPI_Reduce(&subtotalM, &allM, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
1573
1574 printf("total: %d\n", allM);
1575 esl_stopwatch_Stop(w);
1576 esl_stopwatch_Display(stdout, w, "CPU Time: ");
1577 esl_stopwatch_Destroy(w);
1578 }
1579 /* Worker MPI process: */
1580 else
1581 {
1582 P7_PROFILE *gm_recd = NULL;
1583
1584 while (p7_profile_MPIRecv(0, 0, MPI_COMM_WORLD, abc, bg, &buf, &nbuf, &gm_recd) == eslOK)
1585 {
1586 subtotalM += gm_recd->M;
1587 p7_profile_Destroy(gm_recd);
1588 }
1589 MPI_Reduce(&subtotalM, &allM, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
1590 }
1591
1592 free(buf);
1593 p7_bg_Destroy(bg);
1594 esl_alphabet_Destroy(abc);
1595 esl_getopts_Destroy(go);
1596 MPI_Finalize();
1597 exit(0);
1598 }
1599
1600 #endif /*p7MPISUPPORT_BENCHMARK*/
1601 /*---------------------- end, benchmark -------------------------*/
1602
1603
1604 /*****************************************************************
1605 * 6. Unit tests
1606 *****************************************************************/
1607 #ifdef p7MPISUPPORT_TESTDRIVE
1608
1609 static void
utest_HMMSendRecv(int my_rank,int nproc)1610 utest_HMMSendRecv(int my_rank, int nproc)
1611 {
1612 ESL_RANDOMNESS *r = esl_randomness_CreateFast(42);
1613 ESL_ALPHABET *abc = esl_alphabet_Create(eslAMINO);
1614 P7_HMM *hmm = NULL;
1615 P7_HMM *xhmm = NULL;
1616 int M = 200;
1617 char *wbuf = NULL;
1618 int wn = 0;
1619 int i;
1620 char errmsg[eslERRBUFSIZE];
1621
1622 p7_hmm_Sample(r, M, abc, &hmm); /* master and worker's sampled HMMs are identical */
1623
1624 if (my_rank == 0)
1625 {
1626 for (i = 1; i < nproc; i++)
1627 {
1628 ESL_DPRINTF1(("Master: receiving test HMM\n"));
1629 p7_hmm_MPIRecv(MPI_ANY_SOURCE, 0, MPI_COMM_WORLD, &wbuf, &wn, &abc, &xhmm);
1630 ESL_DPRINTF1(("Master: test HMM received\n"));
1631
1632 if (p7_hmm_Validate(xhmm, errmsg, 0.001) != eslOK) p7_Die("hmm validation failed: %s", errmsg);
1633 if (p7_hmm_Compare(hmm, xhmm, 0.001) != eslOK) p7_Die("Received HMM not identical to what was sent");
1634
1635 p7_hmm_Destroy(xhmm);
1636 }
1637 }
1638 else
1639 {
1640 ESL_DPRINTF1(("Worker %d: sending test HMM\n", my_rank));
1641 p7_hmm_MPISend(hmm, 0, 0, MPI_COMM_WORLD, &wbuf, &wn);
1642 ESL_DPRINTF1(("Worker %d: test HMM sent\n", my_rank));
1643 }
1644
1645 p7_hmm_Destroy(hmm);
1646 esl_alphabet_Destroy(abc);
1647 esl_randomness_Destroy(r);
1648 free(wbuf);
1649 return;
1650 }
1651
1652 static void
utest_ProfileSendRecv(int my_rank,int nproc)1653 utest_ProfileSendRecv(int my_rank, int nproc)
1654 {
1655 ESL_RANDOMNESS *r = esl_randomness_CreateFast(42);
1656 ESL_ALPHABET *abc = esl_alphabet_Create(eslAMINO);
1657 P7_HMM *hmm = NULL;
1658 P7_BG *bg = NULL;
1659 P7_PROFILE *gm = NULL;
1660 P7_PROFILE *gm2 = NULL;
1661 int M = 200;
1662 int L = 400;
1663 char *wbuf = NULL;
1664 int wn = 0;
1665 int i;
1666 char errbuf[eslERRBUFSIZE];
1667
1668 p7_hmm_Sample(r, M, abc, &hmm); /* master and worker's sampled profiles are identical */
1669 bg = p7_bg_Create(abc);
1670 gm = p7_profile_Create(hmm->M, abc);
1671 p7_ProfileConfig(hmm, bg, gm, L, p7_LOCAL);
1672 p7_bg_SetLength (bg, L);
1673
1674 if (my_rank == 0)
1675 {
1676 for (i = 1; i < nproc; i++)
1677 {
1678 ESL_DPRINTF1(("Master: receiving test profile\n"));
1679 p7_profile_MPIRecv(MPI_ANY_SOURCE, 0, MPI_COMM_WORLD, abc, bg, &wbuf, &wn, &gm2);
1680 ESL_DPRINTF1(("Master: test profile received\n"));
1681
1682 if (p7_profile_Validate(gm2, errbuf, 0.001) != eslOK) p7_Die("profile validation failed: %s", errbuf);
1683 if (p7_profile_Compare(gm, gm2, 0.001) != eslOK) p7_Die("Received profile not identical to what was sent");
1684
1685 p7_profile_Destroy(gm2);
1686 }
1687 }
1688 else
1689 {
1690 ESL_DPRINTF1(("Worker %d: sending test profile\n", my_rank));
1691 p7_profile_MPISend(gm, 0, 0, MPI_COMM_WORLD, &wbuf, &wn);
1692 ESL_DPRINTF1(("Worker %d: test profile sent\n", my_rank));
1693 }
1694
1695 free(wbuf);
1696 p7_profile_Destroy(gm);
1697 p7_bg_Destroy(bg);
1698 p7_hmm_Destroy(hmm);
1699 esl_alphabet_Destroy(abc);
1700 esl_randomness_Destroy(r);
1701 return;
1702 }
1703
1704
1705
1706 #endif /*p7MPISUPPORT_TESTDRIVE*/
1707 /*---------------------- end, unit tests ------------------------*/
1708
1709
1710 /*****************************************************************
1711 * 7. Test driver.
1712 *****************************************************************/
1713 #ifdef p7MPISUPPORT_TESTDRIVE
1714
1715 /* mpicc -o mpisupport_utest -g -Wall -I../easel -L../easel -I. -L. -Dp7MPISUPPORT_TESTDRIVE mpisupport.c -lhmmer -leasel -lm
1716 * In an MPI environment: (qlogin -pe lam-mpi-tight 2; setenv JOB_ID <jobid>; setenv TMPDIR /tmp/<jobid>....
1717 * mpirun C ./mpisupport_utest
1718 */
1719 #include "esl_getopts.h"
1720
1721 static ESL_OPTIONS options[] = {
1722 /* name type default env range toggles reqs incomp help docgroup*/
1723 { "-h", eslARG_NONE, FALSE, NULL, NULL, NULL, NULL, NULL, "show brief help on version and usage", 0 },
1724 { "--stall", eslARG_NONE, FALSE, NULL, NULL, NULL, NULL, NULL, "arrest after start: for debugging MPI under gdb", 0 },
1725 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
1726 };
1727 static char usage[] = "[-options]";
1728 static char banner[] = "test driver for mpisupport.c";
1729
1730 int
main(int argc,char ** argv)1731 main(int argc, char **argv)
1732 {
1733 ESL_GETOPTS *go = p7_CreateDefaultApp(options, 0, argc, argv, banner, usage);
1734 int stalling = FALSE;
1735 int my_rank;
1736 int nproc;
1737
1738 /* For debugging: stall until GDB can be attached */
1739 if (esl_opt_GetBoolean(go, "--stall")) stalling = TRUE;
1740 while (stalling);
1741
1742 MPI_Init(&argc, &argv);
1743 MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
1744 MPI_Comm_size(MPI_COMM_WORLD, &nproc);
1745
1746 utest_HMMSendRecv(my_rank, nproc);
1747 utest_ProfileSendRecv(my_rank, nproc);
1748
1749 MPI_Finalize();
1750 return 0;
1751 }
1752
1753 #endif /*p7MPISUPPORT_TESTDRIVE*/
1754 /*---------------------- end, test driver -----------------------*/
1755
1756
1757 #else /*!HMMER_MPI*/
1758
1759 /* If we don't have MPI compiled in, provide some nothingness to:
1760 * a. prevent Mac OS/X ranlib from bitching about .o file that "has no symbols"
1761 * b. prevent compiler from bitching about "empty compilation unit"
1762 * c. automatically pass the automated tests.
1763 */
p7_mpisupport_DoAbsolutelyNothing(void)1764 void p7_mpisupport_DoAbsolutelyNothing(void) { return; }
1765 #if defined p7MPISUPPORT_TESTDRIVE || p7MPISUPPORT_EXAMPLE || p7MPISUPPORT_BENCHMARK
main(void)1766 int main(void) { return 0; }
1767 #endif
1768 #endif /*HMMER_MPI*/
1769
1770
1771
1772