1 /** 2 * Copyright (C) Mellanox Technologies Ltd. 2001-2019. ALL RIGHTS RESERVED. 3 * 4 * Copyright (C) UT-Battelle, LLC. 2015. ALL RIGHTS RESERVED. 5 * Copyright (C) ARM Ltd. 2017. ALL RIGHTS RESERVED 6 * Copyright (C) Advanced Micro Devices, Inc. 2016 - 2017. ALL RIGHTS RESERVED. 7 * See file LICENSE for terms. 8 */ 9 10 #ifndef UCT_TEST_H_ 11 #define UCT_TEST_H_ 12 13 #include <common/test.h> 14 15 #include <poll.h> 16 #include <uct/api/uct.h> 17 #include <ucs/sys/sys.h> 18 #include <ucs/async/async.h> 19 #include <ucs/async/pipe.h> 20 #include <common/mem_buffer.h> 21 #include <common/test.h> 22 #include <vector> 23 24 25 26 #define DEFAULT_DELAY_MS 1.0 27 #define DEFAULT_TIMEOUT_SEC 10.0 28 #define DEFAULT_VARIANT 0 29 30 #define UCT_TEST_CALL_AND_TRY_AGAIN(_func, _res) \ 31 do { \ 32 _res = _func; \ 33 if (_res == UCS_ERR_NO_RESOURCE) { \ 34 short_progress_loop(); \ 35 } \ 36 } while (_res == UCS_ERR_NO_RESOURCE) 37 38 39 #define FOR_EACH_ENTITY(_iter) \ 40 for (ucs::ptr_vector<entity>::const_iterator _iter = m_entities.begin(); \ 41 _iter != m_entities.end(); ++_iter) \ 42 43 44 /* Testing resource */ 45 struct resource { ~resourceresource46 virtual ~resource() {}; 47 virtual std::string name() const; 48 uct_component_h component; 49 std::string md_name; 50 ucs_cpu_set_t local_cpus; 51 std::string tl_name; 52 std::string dev_name; 53 std::string variant_name; 54 uct_device_type_t dev_type; 55 ucs::sock_addr_storage listen_sock_addr; /* sockaddr to listen on */ 56 ucs::sock_addr_storage connect_sock_addr; /* sockaddr to connect to */ 57 int variant; 58 59 resource(); 60 resource(uct_component_h component, const std::string& md_name, 61 const ucs_cpu_set_t& local_cpus, const std::string& tl_name, 62 const std::string& dev_name, uct_device_type_t dev_type); 63 resource(uct_component_h component, const uct_md_attr_t& md_attr, 64 const uct_md_resource_desc_t& md_resource, 65 const uct_tl_resource_desc_t& tl_resource); 66 }; 67 68 struct resource_speed : public resource { 69 double bw; 70 resource_speedresource_speed71 resource_speed() : resource(), bw(0) { } 72 resource_speed(uct_component_h component, const uct_worker_h& worker, 73 const uct_md_h& md, const uct_md_attr_t& md_attr, 74 const uct_md_resource_desc_t& md_resource, 75 const uct_tl_resource_desc_t& tl_resource); 76 }; 77 78 79 /** 80 * UCT test, without parameterization 81 */ 82 class uct_test_base : public ucs::test_base { 83 protected: 84 struct md_resource { 85 uct_component_h cmpt; 86 uct_component_attr_t cmpt_attr; 87 uct_md_resource_desc_t rsc_desc; 88 }; 89 90 static std::vector<md_resource> enum_md_resources(); 91 }; 92 93 94 /** 95 * UCT test, parametrized on a transport/device. 96 */ 97 class uct_test : public testing::TestWithParam<const resource*>, 98 public uct_test_base { 99 public: 100 UCS_TEST_BASE_IMPL; 101 102 /* we return a vector of pointers to allow test fixtures to extend the 103 * resource structure. 104 */ 105 static std::vector<const resource*> enum_resources(const std::string& tl_name); 106 107 /* By default generate test variant for all tls. If variant is specific to 108 * the particular transport tl_name need to be specified accordingly */ 109 static void generate_test_variant(int variant, 110 const std::string &variant_name, 111 std::vector<resource>& test_res, 112 const std::string &tl_name=""); 113 uct_test(); 114 virtual ~uct_test(); 115 116 enum atomic_mode { 117 OP32, 118 OP64, 119 FOP32, 120 FOP64 121 }; 122 123 protected: 124 125 class entity { 126 public: 127 typedef uct_test::atomic_mode atomic_mode; 128 typedef std::vector< ucs::handle<uct_ep_h> > eps_vec_t; 129 130 entity(const resource& resource, uct_iface_config_t *iface_config, 131 uct_iface_params_t *params, uct_md_config_t *md_config); 132 133 entity(const resource& resource, uct_md_config_t *md_config, 134 uct_cm_config_t *cm_config); 135 136 void mem_alloc_host(size_t length, uct_allocated_memory_t *mem) const; 137 138 void mem_free_host(const uct_allocated_memory_t *mem) const; 139 140 void mem_type_reg(uct_allocated_memory_t *mem) const; 141 142 void mem_type_dereg(uct_allocated_memory_t *mem) const; 143 144 void rkey_unpack(const uct_allocated_memory_t *mem, 145 uct_rkey_bundle *rkey_bundle) const; 146 147 void rkey_release(const uct_rkey_bundle *rkey_bundle) const; 148 149 unsigned progress() const; 150 151 bool is_caps_supported(uint64_t required_flags); 152 bool check_caps(uint64_t required_flags, uint64_t invalid_flags = 0); 153 bool check_event_caps(uint64_t required_flags, uint64_t invalid_flags = 0); 154 bool check_atomics(uint64_t required_ops, atomic_mode mode); 155 156 uct_md_h md() const; 157 158 const uct_md_attr& md_attr() const; 159 160 uct_worker_h worker() const; 161 162 uct_cm_h cm() const; 163 164 const uct_cm_attr_t& cm_attr() const; 165 166 uct_listener_h listener() const; 167 168 uct_iface_h iface() const; 169 170 const uct_iface_attr& iface_attr() const; 171 172 const uct_iface_params& iface_params() const; 173 174 uct_ep_h ep(unsigned index) const; 175 176 eps_vec_t& eps(); 177 size_t num_eps() const; 178 void reserve_ep(unsigned index); 179 180 void create_ep(unsigned index); 181 void destroy_ep(unsigned index); 182 void revoke_ep(unsigned index); 183 void destroy_eps(); 184 void connect(unsigned index, entity& other, unsigned other_index); 185 void connect(unsigned index, entity& other, unsigned other_index, 186 const ucs::sock_addr_storage &remote_addr, 187 uct_cm_ep_priv_data_pack_callback_t pack_cb, 188 uct_cm_ep_client_connect_callback_t connect_cb, 189 uct_ep_disconnect_cb_t disconnect_cb, 190 void *user_data); 191 void connect_to_iface(unsigned index, entity& other); 192 void connect_to_ep(unsigned index, entity& other, 193 unsigned other_index); 194 void connect_to_sockaddr(unsigned index, entity& other, 195 const ucs::sock_addr_storage &remote_addr, 196 uct_cm_ep_priv_data_pack_callback_t pack_cb, 197 uct_cm_ep_client_connect_callback_t connect_cb, 198 uct_ep_disconnect_cb_t disconnect_cb, 199 void *user_sata); 200 201 void listen(const ucs::sock_addr_storage &listen_addr, 202 const uct_listener_params_t ¶ms); 203 void disconnect(uct_ep_h ep); 204 205 void flush() const; 206 207 size_t max_conn_priv; 208 209 class scoped_async_lock { 210 public: 211 scoped_async_lock(entity &e); 212 ~scoped_async_lock(); 213 private: 214 entity &m_entity; 215 }; 216 217 private: 218 class async_wrapper { 219 public: 220 ucs_async_context_t m_async; 221 async_wrapper(); 222 ~async_wrapper(); 223 void check_miss(); 224 private: 225 async_wrapper(const async_wrapper &); 226 }; 227 228 entity(const entity&); 229 230 231 void connect_p2p_ep(uct_ep_h from, uct_ep_h to); 232 void cuda_mem_alloc(size_t length, uct_allocated_memory_t *mem) const; 233 void cuda_mem_free(const uct_allocated_memory_t *mem) const; 234 235 const resource m_resource; 236 ucs::handle<uct_md_h> m_md; 237 uct_md_attr_t m_md_attr; 238 mutable async_wrapper m_async; 239 ucs::handle<uct_worker_h> m_worker; 240 ucs::handle<uct_cm_h> m_cm; 241 uct_cm_attr_t m_cm_attr; 242 ucs::handle<uct_listener_h> m_listener; 243 ucs::handle<uct_iface_h> m_iface; 244 eps_vec_t m_eps; 245 uct_iface_attr_t m_iface_attr; 246 uct_iface_params_t m_iface_params; 247 }; 248 249 class mapped_buffer { 250 public: 251 mapped_buffer(size_t size, uint64_t seed, const entity& entity, 252 size_t offset = 0, 253 ucs_memory_type_t mem_type = UCS_MEMORY_TYPE_HOST); 254 virtual ~mapped_buffer(); 255 256 void *ptr() const; 257 uintptr_t addr() const; 258 size_t length() const; 259 uct_mem_h memh() const; 260 uct_rkey_t rkey() const; 261 const uct_iov_t* iov() const; 262 263 void pattern_fill(uint64_t seed); 264 void pattern_check(uint64_t seed); 265 266 static size_t pack(void *dest, void *arg); 267 268 private: 269 270 const uct_test::entity& m_entity; 271 272 void *m_buf; 273 void *m_end; 274 uct_rkey_bundle_t m_rkey; 275 uct_allocated_memory_t m_mem; 276 uct_iov_t m_iov; 277 }; 278 279 class async_event_ctx { 280 public: async_event_ctx()281 async_event_ctx() { 282 wakeup_fd.fd = -1; 283 wakeup_fd.events = POLLIN; 284 wakeup_fd.revents = 0; 285 aux_pipe_init = false; 286 memset(&aux_pipe, 0, sizeof(aux_pipe)); 287 } 288 ~async_event_ctx()289 ~async_event_ctx() { 290 if (aux_pipe_init) { 291 ucs_async_pipe_destroy(&aux_pipe); 292 } 293 } 294 295 void signal(); 296 bool wait_for_event(entity &e, int timeout); 297 298 private: 299 struct pollfd wakeup_fd; 300 /* this used for UCT TLs that support async event cb 301 * for event notification */ 302 ucs_async_pipe_t aux_pipe; 303 bool aux_pipe_init; 304 }; 305 306 template <typename T> filter_resources(const std::vector<T> & resources,const std::string & tl_name)307 static std::vector<const resource*> filter_resources(const std::vector<T>& resources, 308 const std::string& tl_name) 309 { 310 std::vector<const resource*> result; 311 for (typename std::vector<T>::const_iterator iter = resources.begin(); 312 iter != resources.end(); ++iter) 313 { 314 if (tl_name.empty() || (iter->tl_name == tl_name)) { 315 result.push_back(&*iter); 316 } 317 } 318 return result; 319 } 320 321 template <typename T> 322 void wait_for_flag(volatile T *flag, double timeout = DEFAULT_TIMEOUT_SEC) const 323 { 324 ucs_time_t deadline = ucs_get_time() + 325 ucs_time_from_sec(timeout) * ucs::test_time_multiplier(); 326 while ((ucs_get_time() < deadline) && (!(*flag))) { 327 short_progress_loop(); 328 } 329 } 330 331 void wait_for_bits(volatile uint64_t *flag, uint64_t mask, 332 double timeout = DEFAULT_TIMEOUT_SEC) const 333 { 334 ucs_time_t deadline = ucs_get_time() + 335 ucs_time_from_sec(timeout) * 336 ucs::test_time_multiplier(); 337 while ((ucs_get_time() < deadline) && (!ucs_test_all_flags(*flag, mask))) { 338 /* Don't do short_progress_loop() to avoid extra timings */ 339 progress(); 340 } 341 } 342 343 template <typename T> 344 void wait_for_value(volatile T *var, T value, bool progress, 345 double timeout = DEFAULT_TIMEOUT_SEC) const 346 { 347 ucs_time_t deadline = ucs_get_time() + 348 ucs_time_from_sec(timeout) * ucs::test_time_multiplier(); 349 while ((ucs_get_time() < deadline) && (*var != value)) { 350 if (progress) { 351 short_progress_loop(); 352 } else { 353 twait(); 354 } 355 } 356 } 357 358 virtual void init(); 359 virtual void cleanup(); 360 virtual void modify_config(const std::string& name, const std::string& value, 361 bool optional = false); 362 bool get_config(const std::string& name, std::string& value) const; 363 void stats_activate(); 364 void stats_restore(); 365 366 virtual bool has_transport(const std::string& tl_name) const; 367 virtual bool has_ud() const; 368 virtual bool has_rc() const; 369 virtual bool has_rc_or_dc() const; 370 virtual bool has_ib() const; 371 372 bool is_caps_supported(uint64_t required_flags); 373 bool check_caps(uint64_t required_flags, uint64_t invalid_flags = 0); 374 void check_caps_skip(uint64_t required_flags, uint64_t invalid_flags = 0); 375 bool check_event_caps(uint64_t required_flags, uint64_t invalid_flags = 0); 376 bool check_atomics(uint64_t required_ops, atomic_mode mode); 377 const entity& ent(unsigned index) const; 378 unsigned progress() const; 379 void flush(ucs_time_t deadline = ULONG_MAX) const; 380 virtual void short_progress_loop(double delay_ms = DEFAULT_DELAY_MS) const; 381 virtual void twait(int delta_ms = DEFAULT_DELAY_MS) const; 382 static void set_cm_resources(std::vector<resource>& all_resources); 383 static bool is_interface_usable(struct ifaddrs *ifa, const char *name); 384 static void set_md_sockaddr_resources(const md_resource& md_rsc, uct_md_h pm, 385 ucs_cpu_set_t local_cpus, 386 std::vector<resource>& all_resources); 387 static void set_cm_sockaddr_resources(uct_component_h cmpt, const char *cmpt_name, 388 ucs_cpu_set_t local_cpus, 389 std::vector<resource>& all_resources); 390 static void set_interface_rscs(uct_component_h comt, const char * name, 391 ucs_cpu_set_t local_cpus, struct ifaddrs *ifa, 392 std::vector<resource>& all_resources); 393 static void init_sockaddr_rsc(resource *rsc, struct sockaddr *listen_addr, 394 struct sockaddr *connect_addr, size_t size); 395 uct_test::entity* create_entity(size_t rx_headroom, 396 uct_error_handler_t err_handler = NULL, 397 uct_tag_unexp_eager_cb_t eager_cb = NULL, 398 uct_tag_unexp_rndv_cb_t rndv_cb = NULL, 399 void *eager_arg = NULL, 400 void *rndv_arg = NULL, 401 uct_async_event_cb_t async_event_cb = NULL, 402 void *async_event_arg = NULL); 403 uct_test::entity* create_entity(uct_iface_params_t ¶ms); 404 uct_test::entity* create_entity(); 405 int max_connections(); 406 int max_connect_batch(); 407 408 void reduce_tl_send_queues(); 409 410 ucs_status_t send_am_message(entity *e, uint8_t am_id = 0, int ep_idx = 0); 411 412 ucs::ptr_vector<entity> m_entities; 413 uct_iface_config_t *m_iface_config; 414 uct_md_config_t *m_md_config; 415 uct_cm_config_t *m_cm_config; 416 }; 417 418 std::ostream& operator<<(std::ostream& os, const resource* resource); 419 420 421 class test_uct_iface_attrs : public uct_test { 422 public: 423 typedef std::map<std::string, size_t> attr_map_t; 424 425 void init(); 426 virtual attr_map_t get_num_iov() = 0; 427 void basic_iov_test(); 428 429 protected: 430 entity *m_e; 431 }; 432 433 434 #define UCT_TEST_IB_TLS \ 435 rc_mlx5, \ 436 rc_verbs, \ 437 dc_mlx5, \ 438 ud_verbs, \ 439 ud_mlx5, \ 440 cm 441 442 #define UCT_TEST_SOCKADDR_TLS \ 443 sockaddr 444 445 #define UCT_TEST_NO_SELF_TLS \ 446 UCT_TEST_IB_TLS, \ 447 ugni_rdma, \ 448 ugni_udt, \ 449 ugni_smsg, \ 450 tcp, \ 451 posix, \ 452 sysv, \ 453 xpmem, \ 454 cma, \ 455 knem 456 457 #define UCT_TEST_CUDA_MEM_TYPE_TLS \ 458 cuda_copy, \ 459 gdr_copy 460 461 #define UCT_TEST_ROCM_MEM_TYPE_TLS \ 462 rocm_copy 463 464 #define UCT_TEST_TLS \ 465 UCT_TEST_NO_SELF_TLS, \ 466 UCT_TEST_CUDA_MEM_TYPE_TLS, \ 467 UCT_TEST_ROCM_MEM_TYPE_TLS, \ 468 self 469 470 /** 471 * Instantiate the parametrized test case for all transports. 472 * 473 * @param _test_case Test case class, derived from uct_test. 474 */ 475 #define UCT_INSTANTIATE_TEST_CASE(_test_case) \ 476 UCS_PP_FOREACH(_UCT_INSTANTIATE_TEST_CASE, _test_case, UCT_TEST_TLS) 477 #define _UCT_INSTANTIATE_TEST_CASE(_test_case, _tl_name) \ 478 INSTANTIATE_TEST_CASE_P(_tl_name, _test_case, \ 479 testing::ValuesIn(_test_case::enum_resources(UCS_PP_QUOTE(_tl_name)))); 480 481 482 /** 483 * Instantiate the parametrized test case for the IB transports. 484 * 485 * @param _test_case Test case class, derived from uct_test. 486 */ 487 #define UCT_INSTANTIATE_IB_TEST_CASE(_test_case) \ 488 UCS_PP_FOREACH(_UCT_INSTANTIATE_TEST_CASE, _test_case, UCT_TEST_IB_TLS) 489 490 /** 491 * Instantiate the parametrized test case for all transports excluding SELF. 492 * 493 * @param _test_case Test case class, derived from uct_test. 494 */ 495 #define UCT_INSTANTIATE_NO_SELF_TEST_CASE(_test_case) \ 496 UCS_PP_FOREACH(_UCT_INSTANTIATE_TEST_CASE, _test_case, UCT_TEST_NO_SELF_TLS) 497 498 #define UCT_INSTANTIATE_SOCKADDR_TEST_CASE(_test_case) \ 499 UCS_PP_FOREACH(_UCT_INSTANTIATE_TEST_CASE, _test_case, UCT_TEST_SOCKADDR_TLS) 500 501 /** 502 * Instantiate the parametrized test case for the RC/DC transports. 503 * 504 * @param _test_case Test case class, derived from uct_test. 505 */ 506 #define UCT_INSTANTIATE_RC_TEST_CASE(_test_case) \ 507 _UCT_INSTANTIATE_TEST_CASE(_test_case, rc_verbs) \ 508 _UCT_INSTANTIATE_TEST_CASE(_test_case, rc_mlx5) 509 510 #define UCT_INSTANTIATE_RC_DC_TEST_CASE(_test_case) \ 511 UCT_INSTANTIATE_RC_TEST_CASE(_test_case) \ 512 _UCT_INSTANTIATE_TEST_CASE(_test_case, dc_mlx5) 513 514 std::ostream& operator<<(std::ostream& os, const uct_tl_resource_desc_t& resource); 515 516 #endif 517