1 // Copyright (c) 2015-2016 John Biddiscombe 2 // Copyright (c) 2013-2015 Thomas Heller 3 // Copyright (c) 2013-2014 Hartmut Kaiser 4 // 5 // Distributed under the Boost Software License, Version 1.0. (See accompanying 6 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 7 8 #ifndef HPX_PARCELSET_POLICIES_LIBFABRIC_HEADER_HPP 9 #define HPX_PARCELSET_POLICIES_LIBFABRIC_HEADER_HPP 10 11 #include <plugins/parcelport/parcelport_logging.hpp> 12 #include <hpx/runtime/parcelset/parcel_buffer.hpp> 13 #include <hpx/util/assert.hpp> 14 // 15 #include <array> 16 #include <cstdint> 17 #include <cstddef> 18 #include <cstring> 19 #include <utility> 20 #include <vector> 21 22 // A generic header structure that can be used by parcelports 23 // currently, the libfabric parcelport makes use of it 24 namespace hpx { 25 namespace parcelset { 26 namespace policies { 27 namespace libfabric 28 { 29 namespace detail 30 { 31 typedef serialization::serialization_chunk chunktype; 32 33 // if chunks are not piggybacked, we must send an rma handle for chunk acccess 34 // and state how many other rma chunks need to be retrieved (since this is 35 // normally stored in the missing chunk info) 36 struct chunk_header { 37 uint32_t num_rma_chunks; 38 chunktype chunk_rma; 39 }; 40 41 // data we send if there are zero copy blocks (or non piggybacked header/chunks) 42 struct rma_info { 43 uint64_t tag; 44 }; 45 46 // data we send if message is piggybacked 47 struct message_info { 48 uint64_t message_size; 49 }; 50 51 // data we send if both message and chunk data are -not- piggybacked 52 // to store the rma information for the message (that otherwise whould be in 53 // the chunk data) 54 struct message_chunk { 55 chunktype message_rma; 56 }; 57 58 // this header block is always sent 59 struct header_block { 60 uint32_t num_chunks; 61 uint32_t flags; // for padding to nice boundary (only need a few bits) 62 }; 63 } 64 65 template <int SIZE> 66 struct header 67 { 68 static constexpr unsigned int header_block_size = sizeof(detail::header_block); 69 static constexpr unsigned int data_size_ = SIZE - header_block_size; 70 // 71 static const unsigned int chunk_flag = 0x01; // chunks piggybacked 72 static const unsigned int message_flag = 0x02; // message pigybacked 73 static const unsigned int normal_flag = 0x04; // normal chunks present 74 static const unsigned int zerocopy_flag = 0x08; // zerocopy chunks present 75 76 typedef serialization::serialization_chunk chunktype; 77 78 private: 79 // 80 // this is the actual header content 81 // 82 detail::header_block message_header; 83 std::array<char, data_size_> data_; 84 // the data block is laid out as follows for each optional item 85 // message_header - always present header_block_size 86 // chunk data : sizeof(chunktype) * numchunks : when chunks piggybacked 87 // or : sizeof(chunk_header) : when chunks not piggybacked 88 // rma_info : sizeof(rma_info) : when we have anything to be rma'd 89 // message_info : sizeof(message_info) : only when message pigybacked 90 // or : sizeof(message_chunk) : when message+chunk both not piggybacked 91 // ..... 92 // message : buffer.size_ : only when message piggybacked 93 94 public: 95 // 96 template <typename Buffer> headerhpx::parcelset::policies::libfabric::header97 header(Buffer const & buffer, void* tag) 98 { 99 const std::vector<chunktype>& chunks = buffer.chunks_; 100 // 101 message_header.flags = 0; 102 message_header.num_chunks = chunks.size(); 103 message_header.flags |= buffer.num_chunks_.first ? zerocopy_flag : 0; 104 message_header.flags |= buffer.num_chunks_.second ? normal_flag : 0; 105 106 // space occupied by chunk data 107 size_t chunkbytes = chunks.size() * sizeof(chunktype); 108 109 // can we send the chunk info inside the header 110 // (NB. we add +1 chunk just in case of a non piggybacked message chunk) 111 if ((chunkbytes+sizeof(chunktype)) <= data_size_) { 112 message_header.flags |= chunk_flag; 113 // copy chunk data directly into the header 114 std::memcpy(&data_[chunk_data_offset()], chunks.data(), chunkbytes); 115 } 116 else 117 { 118 LOG_DEBUG_MSG("Too many chunks for header " 119 << decnumber(chunks.size()) 120 << "requires bytes " << decnumber(chunkbytes)); 121 message_header.flags &= ~chunk_flag; 122 message_header.flags |= zerocopy_flag; 123 // send just rma-get information, address and rma key will be added later 124 detail::chunk_header *ch = 125 reinterpret_cast<detail::chunk_header*>(&data_[chunk_data_offset()]); 126 ch->num_rma_chunks = buffer.num_chunks_.first; 127 ch->chunk_rma = 128 serialization::create_pointer_chunk(nullptr, chunkbytes, 0); 129 // reset chunkbytes size to size of rma hunk header 130 chunkbytes = sizeof(detail::chunk_header); 131 } 132 133 // can we send main message inside the header 134 if (buffer.data_.size() <= (data_size_ - chunkbytes)) { 135 message_header.flags |= message_flag; 136 detail::message_info *info = message_info_ptr(); 137 info->message_size = buffer.size_; 138 } 139 else { 140 message_header.flags &= ~message_flag; 141 message_header.flags |= zerocopy_flag; 142 if ((message_header.flags & chunk_flag) != 0) { 143 // if chunks are piggybacked, just add one rma chunk for the message 144 message_header.num_chunks += 1; 145 chunktype message = 146 serialization::create_pointer_chunk(nullptr, buffer.size_, 0); 147 std::memcpy(&data_[chunkbytes], &message, sizeof(chunktype)); 148 } 149 else { 150 // the message isn't piggybacked and neither is the chunk data 151 // so we must add rma-get information for the message 152 detail::message_chunk *mc = reinterpret_cast<detail::message_chunk*> 153 (&data_[message_info_offset()]); 154 LOG_DEBUG_MSG("Setting chunk free message size to " 155 << decnumber(buffer.size_) 156 << "offset " << decnumber(message_info_offset())); 157 mc->message_rma = 158 serialization::create_pointer_chunk(nullptr, buffer.size_, 0); 159 } 160 } 161 162 // set the rma tag 163 if ((message_header.flags & zerocopy_flag) != 0) { 164 auto ptr = rma_info_ptr(); 165 ptr->tag = reinterpret_cast<uint64_t>(tag); 166 } 167 168 LOG_DEBUG_MSG("Header : " << *this); 169 } 170 171 // -------------------------------------------------------------------- operator <<(std::ostream & os,header<SIZE> & h)172 friend std::ostream & operator<<(std::ostream & os, header<SIZE> & h) 173 { 174 os << "Flags " << hexbyte(h.message_header.flags) 175 << "chunk_data_offset " << decnumber(h.chunk_data_offset()) 176 << "rma_info_offset " << decnumber(h.rma_info_offset()) 177 << "message_info_offset " << decnumber(h.message_info_offset()) 178 << "message_offset " << decnumber(h.message_offset()) 179 << "header length " << decnumber(h.header_length()) 180 << "message length " << hexlength(h.message_size()) 181 << "chunks " << decnumber(h.num_chunks()) 182 << "zerocopy ( " << decnumber(h.num_zero_copy_chunks()) << ") " 183 << "normal ( " << decnumber(h.num_index_chunks()) << ") " 184 << "piggyback " << decnumber((h.message_piggy_back())) 185 << "tag " << hexuint64(h.tag()); 186 return os; 187 } 188 189 public: 190 // ------------------------------------------------------------------ 191 // if chunks are piggybacked, return pointer to list of chunk data chunk_ptrhpx::parcelset::policies::libfabric::header192 inline char *chunk_ptr() 193 { 194 if ((message_header.flags & chunk_flag) == 0) { 195 return nullptr; 196 } 197 return reinterpret_cast<char *>(&data_[chunk_data_offset()]); 198 } 199 200 // ------------------------------------------------------------------ 201 // if chunks are not piggybacked, return pointer to chunk rma info chunk_header_ptrhpx::parcelset::policies::libfabric::header202 inline detail::chunk_header *chunk_header_ptr() 203 { 204 if ((message_header.flags & chunk_flag) == 0) { 205 return reinterpret_cast<detail::chunk_header *> 206 (&data_[chunk_data_offset()]); 207 } 208 return nullptr; 209 } 210 211 // ------------------------------------------------------------------ 212 // if there are rma blocks, return pointer to the rma tag rma_info_ptrhpx::parcelset::policies::libfabric::header213 inline detail::rma_info *rma_info_ptr() 214 { 215 if ((message_header.flags & zerocopy_flag) == 0) { 216 return nullptr; 217 } 218 return reinterpret_cast<detail::rma_info *>(&data_[rma_info_offset()]); 219 } 220 221 // ------------------------------------------------------------------ 222 // if message is piggybacked, return pointer to start of message block message_info_ptrhpx::parcelset::policies::libfabric::header223 inline detail::message_info *message_info_ptr() 224 { 225 if ((message_header.flags & message_flag) == 0) { 226 return nullptr; 227 } 228 return reinterpret_cast<detail::message_info*> 229 (&data_[message_info_offset()]); 230 } 231 232 // ------------------------------------------------------------------ 233 // if message+chunk are not piggybacked, return pointer to message chunk message_chunk_ptrhpx::parcelset::policies::libfabric::header234 inline detail::message_chunk *message_chunk_ptr() 235 { 236 if ((message_header.flags & message_flag) == 0 && 237 (message_header.flags & chunk_flag) == 0) 238 { 239 return reinterpret_cast<detail::message_chunk*> 240 (&data_[message_info_offset()]); 241 } 242 return nullptr; 243 } 244 245 // ------------------------------------------------------------------ message_ptrhpx::parcelset::policies::libfabric::header246 inline char *message_ptr() 247 { 248 if ((message_header.flags & message_flag) == 0) { 249 return nullptr; 250 } 251 return reinterpret_cast<char*>(&data_[message_offset()]); 252 } 253 254 // ------------------------------------------------------------------ chunk_data_offsethpx::parcelset::policies::libfabric::header255 inline uint32_t chunk_data_offset() const 256 { 257 // just in case we ever add any new stuff 258 return 0; 259 } 260 rma_info_offsethpx::parcelset::policies::libfabric::header261 inline uint32_t rma_info_offset() const 262 { 263 // add the chunk data offset 264 std::uint32_t size = chunk_data_offset(); 265 if ((message_header.flags & chunk_flag) !=0) { 266 size = (message_header.num_chunks * sizeof(chunktype)); 267 } 268 else { 269 // chunks are not piggybacked, insert rma details 270 size = sizeof(detail::chunk_header); 271 } 272 return size; 273 } 274 message_info_offsethpx::parcelset::policies::libfabric::header275 inline uint32_t message_info_offset() const 276 { 277 // add the rma info offset 278 std::uint32_t size = rma_info_offset(); 279 if ((message_header.flags & zerocopy_flag) != 0) { 280 size += sizeof(detail::rma_info); 281 } 282 return size; 283 } 284 message_offsethpx::parcelset::policies::libfabric::header285 inline uint32_t message_offset() const 286 { 287 // add the message info offset 288 std::uint32_t size = message_info_offset(); 289 if ((message_header.flags & message_flag) !=0) { 290 size += sizeof(detail::message_info); 291 } 292 else if ((message_header.flags & message_flag) == 0 && 293 (message_header.flags & chunk_flag) == 0) 294 { 295 size += sizeof(detail::message_chunk); 296 } 297 return size; 298 } 299 300 // ------------------------------------------------------------------ 301 // here beginneth the main public API 302 // ------------------------------------------------------------------ chunk_datahpx::parcelset::policies::libfabric::header303 inline char * chunk_data() 304 { 305 return chunk_ptr(); 306 } 307 message_datahpx::parcelset::policies::libfabric::header308 inline char *message_data() 309 { 310 return message_ptr(); 311 } 312 message_piggy_backhpx::parcelset::policies::libfabric::header313 inline bool message_piggy_back() 314 { 315 return message_ptr()!=nullptr; 316 } 317 taghpx::parcelset::policies::libfabric::header318 inline uint64_t tag() 319 { 320 auto ptr = rma_info_ptr(); 321 return ptr ? ptr->tag : 0; 322 } 323 message_sizehpx::parcelset::policies::libfabric::header324 inline uint32_t message_size() 325 { 326 auto ptr = message_info_ptr(); 327 if (ptr) { 328 return ptr->message_size; 329 } 330 // if the data is not piggybacked then look at the final chunk 331 chunktype *chunks = reinterpret_cast<chunktype *>(chunk_ptr()); 332 if (!chunks) { 333 detail::message_chunk *mc = message_chunk_ptr(); 334 LOG_DEBUG_MSG("chunk free message size is " 335 << decnumber(mc->message_rma.size_) 336 << "offset was " << decnumber(message_info_offset())); 337 return mc->message_rma.size_; 338 } 339 return chunks[message_header.num_chunks-1].size_; 340 } 341 342 // the full size of all the header information header_lengthhpx::parcelset::policies::libfabric::header343 inline std::uint32_t header_length() 344 { 345 std::uint32_t size = header_block_size + message_offset(); 346 return size; 347 } 348 set_message_rdma_infohpx::parcelset::policies::libfabric::header349 inline void set_message_rdma_info(uint64_t key, const void *addr) 350 { 351 chunktype *chunks = reinterpret_cast<chunktype *>(chunk_ptr()); 352 if (!chunks) { 353 detail::message_chunk *mc = message_chunk_ptr(); 354 chunks = &mc->message_rma; 355 } 356 else { 357 chunks = &chunks[message_header.num_chunks-1]; 358 } 359 // the last chunk will be our RMA message chunk 360 chunks->rkey_ = key; 361 chunks->data_.cpos_ = addr; 362 } 363 num_chunkshpx::parcelset::policies::libfabric::header364 std::uint32_t num_chunks() 365 { 366 return message_header.num_chunks; 367 } 368 num_zero_copy_chunkshpx::parcelset::policies::libfabric::header369 std::uint32_t num_zero_copy_chunks() 370 { 371 chunktype *chunks = reinterpret_cast<chunktype *>(chunk_ptr()); 372 if (!chunks) { 373 throw std::runtime_error("num_zero_copy_chunks without chunk data"); 374 } 375 uint32_t num=0; 376 for (uint32_t i=0; i<message_header.num_chunks; ++i) { 377 if (chunks[i].type_ == serialization::chunk_type::chunk_type_pointer) { 378 ++num; 379 } 380 } 381 return num; 382 } 383 num_index_chunkshpx::parcelset::policies::libfabric::header384 std::uint32_t num_index_chunks() 385 { 386 chunktype *chunks = reinterpret_cast<chunktype *>(chunk_ptr()); 387 if (!chunks) { 388 throw std::runtime_error("num_index_chunks without chunk data"); 389 } 390 uint32_t num=0; 391 for (uint32_t i=0; i<message_header.num_chunks; ++i) { 392 if (chunks[i].type_ == serialization::chunk_type::chunk_type_index) { 393 ++num; 394 } 395 } 396 return num; 397 } 398 399 }; 400 401 }}}} 402 403 #endif 404