1 /** 2 * Copyright (c) UT-Battelle, LLC. 2014-2017. ALL RIGHTS RESERVED. 3 * Copyright (c) Los Alamos National Security, LLC. 2018. ALL RIGHTS RESERVED. 4 * Copyright (c) Triad National Security, LLC. 2018. ALL RIGHTS RESERVED. 5 * See file LICENSE for terms. 6 */ 7 8 #ifndef UCT_UGNI_TYPES_H 9 #define UCT_UGNI_TYPES_H 10 11 #include "ugni_def.h" 12 13 #include <uct/base/uct_iface.h> 14 #include <uct/base/uct_md.h> 15 #include <ucs/datastruct/arbiter.h> 16 #include <gni_pub.h> 17 18 19 typedef struct uct_ugni_device { 20 gni_nic_device_t type; /**< Device type */ 21 char type_name[UCT_UGNI_MAX_TYPE_NAME]; /**< Device type name */ 22 char fname[UCT_DEVICE_NAME_MAX];/**< Device full name */ 23 uint32_t device_id; /**< Device id */ 24 uint32_t address; /**< Device address */ 25 uint32_t cpu_id; /**< CPU attached directly 26 to the device */ 27 ucs_sys_cpuset_t cpu_mask; /**< CPU mask */ 28 /* TBD - reference counter */ 29 } uct_ugni_device_t; 30 31 typedef struct uct_ugni_cdm { 32 gni_cdm_handle_t cdm_handle; /**< Ugni communication domain */ 33 gni_nic_handle_t nic_handle; /**< Ugni NIC handle */ 34 uct_ugni_device_t *dev; /**< Ugni device the cdm is connected to */ 35 ucs_thread_mode_t thread_mode; 36 uint32_t address; 37 uint32_t domain_id; 38 39 #if ENABLE_MT 40 ucs_recursive_spinlock_t lock; /**< Device lock */ 41 #endif 42 } uct_ugni_cdm_t; 43 44 /** 45 * @brief UGNI Memory domain 46 * 47 * Ugni does not define MD, instead I use 48 * device handle that "simulates" the MD. 49 * Memory that is registered with one device handle 50 * can be accessed with any other. 51 */ 52 typedef struct uct_ugni_md { 53 uct_md_t super; /**< Domain info */ 54 uct_ugni_cdm_t cdm; /**< Communication domain for memory registration*/ 55 int ref_count; /**< UGNI Domain ref count */ 56 } uct_ugni_md_t; 57 58 typedef struct uct_devaddr_ugni_t { 59 uint32_t nic_addr; 60 } UCS_S_PACKED uct_devaddr_ugni_t; 61 62 typedef struct uct_sockaddr_ugni { 63 uint32_t domain_id; 64 } UCS_S_PACKED uct_sockaddr_ugni_t; 65 66 typedef struct uct_ugni_flush_group { 67 uct_completion_t flush_comp; /**< Completion for outstanding requests 68 flush_comp.count is used to track outstanding sends*/ 69 uct_completion_t *user_comp; /**< User completion struct */ 70 struct uct_ugni_flush_group *parent; /**< Used to signal the next flush_group that this group is done*/ 71 } uct_ugni_flush_group_t; 72 73 typedef struct uct_ugni_ep { 74 uct_base_ep_t super; 75 gni_ep_handle_t ep; /**< Endpoint for ugni api */ 76 uct_ugni_flush_group_t *flush_group; /**< Flush group new sends are added to */ 77 uint32_t hash_key; /**< Hash to look up EPs with */ 78 uint32_t arb_sched; /**< Flag to make sure we don't recursively block sends*/ 79 ucs_arbiter_group_t arb_group; /**< Our group in the pending send arbiter */ 80 struct uct_ugni_ep *next; 81 } uct_ugni_ep_t; 82 83 typedef struct uct_ugni_iface { 84 uct_base_iface_t super; 85 uct_ugni_cdm_t cdm; /**< Ugni communication domain and handles */ 86 gni_cq_handle_t local_cq; /**< Completion queue */ 87 uct_ugni_ep_t *eps[UCT_UGNI_HASH_SIZE]; /**< Array of QPs */ 88 unsigned outstanding; /**< Counter for outstanding packets 89 on the interface */ 90 ucs_arbiter_t arbiter; /**< arbiter structure for pending operations */ 91 ucs_mpool_t flush_pool; /**< Memory pool for flush objects */ 92 } uct_ugni_iface_t; 93 94 typedef struct uct_ugni_iface_config { 95 uct_iface_config_t super; 96 uct_iface_mpool_config_t mpool; 97 } uct_ugni_iface_config_t; 98 99 #endif 100