1 /*
2    Copyright (c) 2009-2014, Jack Poulson
3    Copyright (c) 2011, The University of Texas at Austin
4    All rights reserved.
5 
6    Authors:
7    This interface is mainly due to Martin Schatz, but it was put into its
8    current form by Jack Poulson.
9 
10    This file is part of Elemental and is under the BSD 2-Clause License,
11    which can be found in the LICENSE file in the root directory, or at
12    http://opensource.org/licenses/BSD-2-Clause
13 */
14 #pragma once
15 #ifndef ELEM_AXPYINTERFACE_DECL_HPP
16 #define ELEM_AXPYINTERFACE_DECL_HPP
17 
18 namespace elem {
19 
20 namespace AxpyTypeNS {
21 enum AxpyType { LOCAL_TO_GLOBAL, GLOBAL_TO_LOCAL };
22 }
23 using namespace AxpyTypeNS;
24 
25 template<typename T>
26 class AxpyInterface
27 {
28 public:
29     AxpyInterface();
30     ~AxpyInterface();
31 
32     AxpyInterface( AxpyType type,       DistMatrix<T,MC,MR>& Z );
33     AxpyInterface( AxpyType type, const DistMatrix<T,MC,MR>& Z );
34 
35     void Attach( AxpyType type,       DistMatrix<T,MC,MR>& Z );
36     void Attach( AxpyType type, const DistMatrix<T,MC,MR>& Z );
37 
38     void Axpy( T alpha,       Matrix<T>& Z, Int i, Int j );
39     void Axpy( T alpha, const Matrix<T>& Z, Int i, Int j );
40 
41     void Detach();
42 
43 private:
44     static const Int
45         DATA_TAG        =1,
46         EOM_TAG         =2,
47         DATA_REQUEST_TAG=3,
48         DATA_REPLY_TAG  =4;
49 
50     bool attachedForLocalToGlobal_, attachedForGlobalToLocal_;
51     DistMatrix<T,MC,MR>* localToGlobalMat_;
52     const DistMatrix<T,MC,MR>* globalToLocalMat_;
53 
54     byte sendDummy_, recvDummy_;
55 
56     std::vector<bool> sentEomTo_, haveEomFrom_;
57     std::vector<byte> recvVector_;
58     std::vector<mpi::Request> eomSendRequests_;
59 
60     std::vector<std::deque<std::vector<byte>>>
61         dataVectors_, requestVectors_, replyVectors_;
62     std::vector<std::deque<bool>>
63         sendingData_, sendingRequest_, sendingReply_;
64     std::vector<std::deque<mpi::Request>>
65         dataSendRequests_, requestSendRequests_, replySendRequests_;
66 
67     // Check if we are done with this attachment's work
68     bool Finished();
69 
70     // Progress functions
71     void UpdateRequestStatuses();
72     void HandleEoms();
73     void HandleLocalToGlobalData();
74     void HandleGlobalToLocalRequest();
75     void StartSendingEoms();
76     void FinishSendingEoms();
77 
78     void AxpyLocalToGlobal( T alpha, const Matrix<T>& X, Int i, Int j );
79     void AxpyGlobalToLocal( T alpha,       Matrix<T>& Y, Int i, Int j );
80 
81     Int ReadyForSend
82     ( Int sendSize,
83       std::deque<std::vector<byte>>& sendVectors,
84       std::deque<mpi::Request>& requests,
85       std::deque<bool>& requestStatuses );
86 };
87 
88 } // namespace elem
89 
90 #endif // ifndef ELEM_AXPYINTERFACE_DECL_HPP
91