1# Support for the Message Passing Interface (MPI)
2#
3# (C) Copyright 2005, 2006 Trustees of Indiana University
4# (C) Copyright 2005 Douglas Gregor
5#
6# Distributed under the Boost Software License, Version 1.0. (See accompanying
7# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt.)
8#
9# Authors: Douglas Gregor
10#          Andrew Lumsdaine
11#
12# ==== MPI Configuration ====
13#
14# For many users, MPI support can be enabled simply by adding the following
15# line to your user-config.jam file:
16#
17#   using mpi ;
18#
19# This should auto-detect MPI settings based on the MPI wrapper compiler in
20# your path, e.g., "mpic++". If the wrapper compiler is not in your path, or
21# has a different name, you can pass the name of the wrapper compiler as the
22# first argument to the mpi module:
23#
24#   using mpi : /opt/mpich2-1.0.4/bin/mpiCC ;
25#
26# If your MPI implementation does not have a wrapper compiler, or the MPI
27# auto-detection code does not work with your MPI's wrapper compiler,
28# you can pass MPI-related options explicitly via the second parameter to the
29# mpi module:
30#
31#    using mpi : : <find-shared-library>lammpio <find-shared-library>lammpi++
32#                  <find-shared-library>mpi <find-shared-library>lam
33#                  <find-shared-library>dl ;
34#
35# To see the results of MPI auto-detection, pass "--debug-configuration" on
36# the bjam command line.
37#
38# The (optional) fourth argument configures Boost.MPI for running
39# regression tests. These parameters specify the executable used to
40# launch jobs (default: "mpirun") followed by any necessary arguments
41# to this to run tests and tell the program to expect the number of
42# processors to follow (default: "-np").  With the default parameters,
43# for instance, the test harness will execute, e.g.,
44#
45#    mpirun -np 4 all_gather_test
46#
47# ==== Linking Against the MPI Libraries ===
48#
49# To link against the MPI libraries, import the "mpi" module and add the
50# following requirement to your target:
51#
52#   <library>/mpi//mpi
53#
54# Since MPI support is not always available, you should check
55# "mpi.configured" before trying to link against the MPI libraries.
56
57import "class" : new ;
58import common ;
59import feature : feature ;
60import generators ;
61import os ;
62import project ;
63import property ;
64import testing ;
65import toolset ;
66import type ;
67import path ;
68
69# Make this module a project
70project.initialize $(__name__) ;
71project mpi ;
72
73if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
74{
75  .debug-configuration = true ;
76}
77
78# Assuming the first part of the command line is the given prefix
79# followed by some non-empty value, remove the first argument. Returns
80# either nothing (if there was no prefix or no value) or a pair
81#
82#   <name>value rest-of-cmdline
83#
84# This is a subroutine of cmdline_to_features
85rule add_feature ( prefix name cmdline )
86{
87    local match = [ MATCH "^$(prefix)([^\" ]+|\"[^\"]+\") *(.*)$" : $(cmdline) ] ;
88
89    # If there was no value associated with the prefix, abort
90    if ! $(match) {
91      return ;
92    }
93
94    local value = $(match[1]) ;
95
96    if [ MATCH " +" : $(value) ] {
97      value = "\"$(value)\"" ;
98    }
99
100    return "<$(name)>$(value)" $(match[2]) ;
101}
102
103# Strip any end-of-line characters off the given string and return the
104# result.
105rule strip-eol ( string )
106{
107  local match = [ MATCH "^(([A-Za-z0-9~`\.!@#$%^&*()_+={};:'\",.<>/?\\| -]|[|])*).*$" : $(string) ] ;
108
109  if $(match)
110  {
111    return $(match[1]) ;
112  }
113  else
114  {
115    return $(string) ;
116  }
117}
118
119# Split a command-line into a set of features. Certain kinds of
120# compiler flags are recognized (e.g., -I, -D, -L, -l) and replaced
121# with their Boost.Build equivalents (e.g., <include>, <define>,
122# <library-path>, <find-library>). All other arguments are introduced
123# using the features in the unknown-features parameter, because we
124# don't know how to deal with them. For instance, if your compile and
125# correct. The incoming command line should be a string starting with
126# an executable (e.g., g++ -I/include/path") and may contain any
127# number of command-line arguments thereafter. The result is a list of
128# features corresponding to the given command line, ignoring the
129# executable.
130rule cmdline_to_features ( cmdline : unknown-features ? )
131{
132    local executable ;
133    local features ;
134    local otherflags ;
135    local result ;
136
137    unknown-features ?= <cxxflags> <linkflags> ;
138
139    # Pull the executable out of the command line. At this point, the
140    # executable is just thrown away.
141    local match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$" : $(cmdline) ] ;
142    executable = $(match[1]) ;
143    cmdline = $(match[2]) ;
144
145    # List the prefix/feature pairs that we will be able to transform.
146    # Every kind of parameter not mentioned here will be placed in both
147    # cxxflags and linkflags, because we don't know where they should go.
148    local feature_kinds-D = "define" ;
149    local feature_kinds-I = "include" ;
150    local feature_kinds-L = "library-path" ;
151    local feature_kinds-l = "find-shared-library" ;
152
153    while $(cmdline) {
154
155        # Check for one of the feature prefixes we know about. If we
156        # find one (and the associated value is nonempty), convert it
157        # into a feature.
158        local match = [ MATCH "^(-.)(.*)" : $(cmdline) ] ;
159        local matched ;
160        if $(match) && $(match[2]) {
161           local prefix = $(match[1]) ;
162           if $(feature_kinds$(prefix)) {
163               local name = $(feature_kinds$(prefix)) ;
164               local add = [ add_feature $(prefix) $(name) $(cmdline) ] ;
165
166               if $(add) {
167
168                  if $(add[1]) = <find-shared-library>pthread
169                  {
170                      # Uhm. It's not really nice that this MPI implementation
171                      # uses -lpthread as opposed to -pthread. We do want to
172                      # set <threading>multi, instead of -lpthread.
173                      result += "<threading>multi" ;
174                      MPI_EXTRA_REQUIREMENTS += "<threading>multi" ;
175                  }
176                  else
177                  {
178                      result += $(add[1]) ;
179                  }
180
181                  cmdline = $(add[2]) ;
182                  matched = yes ;
183               }
184           }
185        }
186
187        # If we haven't matched a feature prefix, just grab the command-line
188        # argument itself. If we can map this argument to a feature
189        # (e.g., -pthread -> <threading>multi), then do so; otherwise,
190        # and add it to the list of "other" flags that we don't
191        # understand.
192        if ! $(matched) {
193           match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$" : $(cmdline) ] ;
194           local value = $(match[1]) ;
195           cmdline = $(match[2]) ;
196
197           # Check for multithreading support
198           if $(value) = "-pthread" || $(value) = "-pthreads"
199           {
200             result += "<threading>multi" ;
201
202             # DPG: This is a hack intended to work around a BBv2 bug where
203             # requirements propagated from libraries are not checked for
204             # conflicts when BBv2 determines which "common" properties to
205             # apply to a target. In our case, the <threading>single property
206             # gets propagated from the common properties to Boost.MPI
207             # targets, even though <threading>multi is in the usage
208             # requirements of <library>/mpi//mpi.
209             MPI_EXTRA_REQUIREMENTS += "<threading>multi" ;
210           }
211           else if [ MATCH "(.*[a-zA-Z0-9<>?-].*)" : $(value) ] {
212              otherflags += $(value) ;
213           }
214        }
215    }
216
217    # If there are other flags that we don't understand, add them to the
218    # result as both <cxxflags> and <linkflags>
219    if $(otherflags) {
220       for unknown in $(unknown-features)
221       {
222         result += "$(unknown)$(otherflags:J= )" ;
223       }
224    }
225
226    return $(result) ;
227}
228
229# Determine if it is safe to execute the given shell command by trying
230# to execute it and determining whether the exit code is zero or
231# not. Returns true for an exit code of zero, false otherwise.
232local rule safe-shell-command ( cmdline )
233{
234  local result = [ SHELL "$(cmdline) > /dev/null 2>/dev/null; if [ "$?" -eq "0" ]; then echo SSCOK; fi" ] ;
235  return [ MATCH ".*(SSCOK).*" : $(result) ] ;
236}
237
238# Initialize the MPI module.
239rule init ( mpicxx ? : options * : mpirun-with-options * )
240{
241  if ! $(options) && $(.debug-configuration)
242  {
243    ECHO "===============MPI Auto-configuration===============" ;
244  }
245
246  if ! $(mpicxx) && [ os.on-windows ]
247  {
248    # Try to auto-configure to the Microsoft Compute Cluster Pack
249    local cluster_pack_path_native = "C:\\Program Files\\Microsoft Compute Cluster Pack" ;
250    local cluster_pack_path = [ path.make $(cluster_pack_path_native) ] ;
251    if [ GLOB $(cluster_pack_path_native)\\Include : mpi.h ]
252    {
253      if $(.debug-configuration)
254      {
255        ECHO "Found Microsoft Compute Cluster Pack: $(cluster_pack_path_native)" ;
256      }
257
258      # Pick up either the 32-bit or 64-bit library, depending on which address
259      # model the user has selected. Default to 32-bit.
260      options = <include>$(cluster_pack_path)/Include
261                <address-model>64:<library-path>$(cluster_pack_path)/Lib/amd64
262                <library-path>$(cluster_pack_path)/Lib/i386
263                <find-static-library>msmpi
264                <toolset>msvc:<define>_SECURE_SCL=0
265              ;
266
267      # Setup the "mpirun" equivalent (mpiexec)
268      .mpirun = "\"$(cluster_pack_path_native)\\Bin\\mpiexec.exe"\" ;
269      .mpirun_flags = -n ;
270    }
271    else if $(.debug-configuration)
272    {
273      ECHO "Did not find Microsoft Compute Cluster Pack in $(cluster_pack_path_native)." ;
274    }
275  }
276
277  if ! $(options)
278  {
279    # Try to auto-detect options based on the wrapper compiler
280    local command = [ common.get-invocation-command mpi : mpic++ : $(mpicxx) ] ;
281
282    if ! $(mpicxx) && ! $(command)
283    {
284      # Try "mpiCC", which is used by MPICH
285      command = [ common.get-invocation-command mpi : mpiCC ] ;
286    }
287
288    if ! $(mpicxx) && ! $(command)
289    {
290      # Try "mpicxx", which is used by OpenMPI and MPICH2
291      command = [ common.get-invocation-command mpi : mpicxx ] ;
292    }
293
294    if ! $(mpicxx) && ! $(command)
295    {
296      # Try "CC", which is used by Cray
297      command = [ common.get-invocation-command mpi : CC ] ;
298    }
299
300    local result ;
301    local compile_flags ;
302    local link_flags ;
303
304    if ! $(command)
305    {
306      # Do nothing: we'll complain later
307    }
308    # OpenMPI and newer versions of LAM-MPI have -showme:compile and
309    # -showme:link.
310    else if [ safe-shell-command "$(command) -showme:compile" ] &&
311              [ safe-shell-command "$(command) -showme:link" ]
312    {
313      if $(.debug-configuration)
314      {
315        ECHO "Found recent LAM-MPI or Open MPI wrapper compiler: $(command)" ;
316      }
317
318      compile_flags = [ SHELL "$(command) -showme:compile" ] ;
319      link_flags = [ SHELL "$(command) -showme:link" ] ;
320
321      # Prepend COMPILER as the executable name, to match the format of
322      # other compilation commands.
323      compile_flags = "COMPILER $(compile_flags) -DOMPI_SKIP_MPICXX " ;
324      link_flags = "COMPILER $(link_flags)" ;
325    }
326    # Look for LAM-MPI's -showme
327    else if [ safe-shell-command "$(command) -showme" ]
328    {
329      if $(.debug-configuration)
330      {
331        ECHO "Found older LAM-MPI wrapper compiler: $(command)" ;
332      }
333
334      result = [ SHELL "$(command) -showme" ] ;
335    }
336    # Look for MPICH
337    else if [ safe-shell-command "$(command) -show" ]
338    {
339      if $(.debug-configuration)
340      {
341        ECHO "Found MPICH wrapper compiler: $(command)" ;
342      }
343      compile_flags = [ SHELL "$(command) -compile_info" ] ;
344      link_flags = [ SHELL "$(command) -link_info" ] ;
345    }
346    # Sun HPC and Ibm POE
347    else if [ SHELL "$(command) -v 2>/dev/null" ]
348    {
349      compile_flags = [ SHELL "$(command) -c -v -xtarget=native64 2>/dev/null" ] ;
350
351      local back = [ MATCH "--------------------(.*)" : $(compile_flags) ] ;
352      if $(back)
353      {
354        # Sun HPC
355        if $(.debug-configuration)
356        {
357          ECHO "Found Sun MPI wrapper compiler: $(command)" ;
358        }
359
360        compile_flags = [ MATCH "(.*)--------------------" : $(back) ] ;
361        compile_flags = [ MATCH "(.*)-v" :  $(compile_flags) ] ;
362        link_flags = [ SHELL "$(command) -v -xtarget=native64 2>/dev/null" ] ;
363        link_flags = [ MATCH "--------------------(.*)" : $(link_flags) ] ;
364        link_flags = [ MATCH "(.*)--------------------" : $(link_flags) ] ;
365
366        # strip out -v from compile options
367        local front = [ MATCH "(.*)-v" :  $(link_flags) ] ;
368        local back = [ MATCH "-v(.*)" :  $(link_flags) ] ;
369        link_flags = "$(front) $(back)" ;
370        front = [ MATCH "(.*)-xtarget=native64" :  $(link_flags) ] ;
371        back = [ MATCH "-xtarget=native64(.*)" :  $(link_flags) ] ;
372        link_flags = "$(front) $(back)" ;
373      }
374      else
375      {
376        # Ibm POE
377        if $(.debug-configuration)
378        {
379          ECHO "Found IBM MPI wrapper compiler: $(command)" ;
380        }
381
382        #
383        compile_flags = [ SHELL "$(command) -c -v 2>/dev/null" ] ;
384        compile_flags = [ MATCH "(.*)exec: export.*" : $(compile_flags) ] ;
385        local front = [ MATCH "(.*)-v" :  $(compile_flags) ] ;
386        local back = [ MATCH "-v(.*)" :  $(compile_flags) ] ;
387        compile_flags = "$(front) $(back)" ;
388        front = [ MATCH "(.*)-c" :  $(compile_flags) ] ;
389        back = [ MATCH "-c(.*)" :  $(compile_flags) ] ;
390        compile_flags = "$(front) $(back)" ;
391        link_flags = $(compile_flags) ;
392
393        # get location of mpif.h from mpxlf
394        local f_flags = [ SHELL "mpxlf -v 2>/dev/null" ] ;
395        f_flags = [ MATCH "(.*)exec: export.*" : $(f_flags) ] ;
396        front = [ MATCH "(.*)-v" :  $(f_flags) ] ;
397        back = [ MATCH "-v(.*)" :  $(f_flags) ] ;
398        f_flags = "$(front) $(back)" ;
399        f_flags = [ MATCH "xlf_r(.*)" : $(f_flags) ] ;
400        f_flags = [ MATCH "-F:mpxlf_r(.*)" : $(f_flags) ] ;
401        compile_flags = [ strip-eol $(compile_flags) ] ;
402        compile_flags = "$(compile_flags) $(f_flags)" ;
403      }
404    }
405    # Cray
406    else if [ safe-shell-command "$(command) -v" ]
407    {
408      compile_flags = [ safe-shell-command  "$(command) -###" ] ;
409      link_flags = [ safe-shell-command  "$(command) -###" ] ;
410      # ECHO "Noel: compile_flags: $(compile_flags)" ;
411      # ECHO "Noel: link_flags: $(link_flags)" ;
412      result = " " ;
413    }
414
415      # Prepend COMPILER as the executable name, to match the format of
416
417    if $(result) || $(compile_flags) && $(link_flags)
418    {
419      if $(result)
420      {
421         result = [ strip-eol $(result) ] ;
422         options = [ cmdline_to_features $(result) ] ;
423      }
424      else
425      {
426         compile_flags = [ strip-eol $(compile_flags) ] ;
427         link_flags = [ strip-eol $(link_flags) ] ;
428
429         # Separately process compilation and link features, then combine
430         # them at the end.
431         local compile_features = [ cmdline_to_features $(compile_flags)
432                                                        : "<cxxflags>" ] ;
433         local link_features = [ cmdline_to_features $(link_flags)
434                                                     : "<linkflags>" ] ;
435         options = $(compile_features) $(link_features) ;
436      }
437
438      # If requested, display MPI configuration information.
439      if $(.debug-configuration)
440      {
441        if $(result)
442        {
443          ECHO "  Wrapper compiler command line: $(result)" ;
444        }
445        else
446        {
447	  local match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$"
448                                : $(compile_flags) ] ;
449          ECHO "MPI compilation flags: $(match[2])" ;
450	  local match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$"
451                                : $(link_flags) ] ;
452          ECHO "MPI link flags: $(match[2])" ;
453        }
454      }
455    }
456    else
457    {
458      if $(command)
459      {
460        ECHO "MPI auto-detection failed: unknown wrapper compiler $(command)" ;
461        ECHO "Please report this error to the Boost mailing list: http://www.boost.org" ;
462      }
463      else if $(mpicxx)
464      {
465        ECHO "MPI auto-detection failed: unable to find wrapper compiler $(mpicxx)" ;
466      }
467      else
468      {
469        ECHO "MPI auto-detection failed: unable to find wrapper compiler `mpic++' or `mpiCC'" ;
470      }
471      ECHO "You will need to manually configure MPI support." ;
472    }
473
474  }
475
476  # Find mpirun (or its equivalent) and its flags
477  if ! $(.mpirun)
478  {
479    .mpirun =
480        [ common.get-invocation-command mpi : mpirun : $(mpirun-with-options[1]) ] ;
481    .mpirun_flags = $(mpirun-with-options[2-]) ;
482    .mpirun_flags ?= -np ;
483  }
484
485  if $(.debug-configuration)
486  {
487    if $(options)
488    {
489      echo "MPI build features: " ;
490      ECHO $(options) ;
491    }
492
493    if $(.mpirun)
494    {
495      echo "MPI launcher: $(.mpirun) $(.mpirun_flags)" ;
496    }
497
498    ECHO "====================================================" ;
499  }
500
501  if $(options)
502  {
503    .configured = true ;
504
505    # Set up the "mpi" alias
506    alias mpi : : : : $(options) ;
507  }
508}
509
510# States whether MPI has bee configured
511rule configured ( )
512{
513  return $(.configured) ;
514}
515
516# Returs the "extra" requirements needed to build MPI. These requirements are
517# part of the /mpi//mpi library target, but they need to be added to anything
518# that uses MPI directly to work around bugs in BBv2's propagation of
519# requirements.
520rule extra-requirements ( )
521{
522  return $(MPI_EXTRA_REQUIREMENTS) ;
523}
524
525# Support for testing; borrowed from Python
526type.register RUN_MPI_OUTPUT ;
527type.register RUN_MPI : : TEST ;
528
529class mpi-test-generator : generator
530{
531    import property-set ;
532
533    rule __init__ ( * : * )
534    {
535        generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
536        self.composing = true ;
537    }
538
539    rule run ( project name ? : property-set : sources * : multiple ? )
540    {
541      # Generate an executable from the sources. This is the executable we will run.
542      local executable =
543        [ generators.construct $(project) $(name) : EXE : $(property-set) : $(sources) ] ;
544
545      result =
546        [ construct-result $(executable[2-]) : $(project) $(name)-run : $(property-set) ] ;
547    }
548}
549
550# Use mpi-test-generator to generate MPI tests from sources
551generators.register
552  [ new mpi-test-generator mpi.capture-output : : RUN_MPI_OUTPUT ] ;
553
554generators.register-standard testing.expect-success
555  : RUN_MPI_OUTPUT : RUN_MPI ;
556
557# The number of processes to spawn when executing an MPI test.
558feature mpi:processes : : free incidental ;
559
560# The flag settings on testing.capture-output do not
561# apply to mpi.capture output at the moment.
562# Redo this explicitly.
563toolset.flags mpi.capture-output ARGS <testing.arg> ;
564rule capture-output ( target : sources * : properties * )
565{
566    # Use the standard capture-output rule to run the tests
567    testing.capture-output $(target) : $(sources[1]) : $(properties) ;
568
569    # Determine the number of processes we should run on.
570    local num_processes = [ property.select <mpi:processes> : $(properties) ] ;
571    num_processes = $(num_processes:G=) ;
572
573    # serialize the MPI tests to avoid overloading systems
574    JAM_SEMAPHORE on $(target) = <s>mpi-run-semaphore ;
575
576    # We launch MPI processes using the "mpirun" equivalent specified by the user.
577    LAUNCHER on $(target) =
578      [ on $(target) return $(.mpirun) $(.mpirun_flags) $(num_processes) ] ;
579}
580
581# Creates a set of test cases to be run through the MPI launcher. The name, sources,
582# and requirements are the same as for any other test generator. However, schedule is
583# a list of numbers, which indicates how many processes each test run will use. For
584# example, passing 1 2 7 will run the test with 1 process, then 2 processes, then 7
585# 7 processes. The name provided is just the base name: the actual tests will be
586# the name followed by a hypen, then the number of processes.
587rule mpi-test ( name : sources * : requirements * : schedule * )
588{
589    sources ?= $(name).cpp ;
590    schedule ?= 1 2 3 4 7 8 13 17 ;
591
592    local result ;
593    for processes in $(schedule)
594    {
595      result += [ testing.make-test
596        run-mpi : $(sources) /boost/mpi//boost_mpi
597          : $(requirements) <toolset>msvc:<link>static <mpi:processes>$(processes) : $(name)-$(processes) ] ;
598    }
599    return $(result) ;
600}
601