1 /*
2  * Copyright (C) by Argonne National Laboratory
3  *     See COPYRIGHT in top-level directory
4  */
5 
6 #ifndef YAKSURI_ZEI_H_INCLUDED
7 #define YAKSURI_ZEI_H_INCLUDED
8 
9 #include "yaksi.h"
10 #include <stdint.h>
11 #include <pthread.h>
12 #include <level_zero/ze_api.h>
13 #include "yaksuri_zei_md.h"
14 
15 #define ZE_P2P_ENABLED  (1)
16 #define ZE_P2P_DISABLED (2)
17 #define ZE_P2P_CLIQUES  (3)
18 
19 #define YAKSURI_KERNEL_NULL   -1
20 
21 #define ZE_THROTTLE_THRESHOLD  (4 * 1024)
22 #define ZE_EVENT_POOL_CAP      (6 * 1024)
23 #define ZE_CMD_LIST_INIT_POOL_SIZE      8
24 
25 #define ZE_DEBUG 0
26 
27 #define YAKSURI_ZEI_INFO__DEFAULT_IOV_PUP_THRESHOLD     (16384)
28 
29 /* *INDENT-OFF* */
30 #ifdef __cplusplus
31 extern "C" {
32 #endif
33 /* *INDENT-ON* */
34 
35 #define YAKSURI_ZEI_ZE_ERR_CHECK(zerr)                                  \
36     do {                                                                \
37         if (zerr != ZE_RESULT_SUCCESS) {                                \
38             fprintf(stderr, "ZE Error (%s:%s,%d): %d\n", __func__, __FILE__, __LINE__, zerr); \
39         }                                                               \
40     } while (0)
41 
42 #define YAKSURI_ZEI_ZE_ERR_CHKANDJUMP(zerr, rc, fn_fail)                \
43     do {                                                                \
44         if (zerr != ZE_RESULT_SUCCESS) {                                \
45             fprintf(stderr, "ZE Error (%s:%s,%d): %d\n", __func__, __FILE__, __LINE__, zerr); \
46             rc = YAKSA_ERR__INTERNAL;                                   \
47             goto fn_fail;                                               \
48         }                                                               \
49     } while (0)
50 
51 typedef struct {
52     ze_event_pool_handle_t ep;  /* event pool, one per device */
53     int ev_pool_idx;
54     int ev_lb, ev_ub;           /* [lb,ub] defines the events being used */
55     ze_event_handle_t *events;
56     int last_event_idx;         /* immed. cmd lists are serialized */
57     ze_command_list_handle_t *cl;       /* immed. cmd lists being used */
58     int num_cl;                 /* number of immed. cmd lists */
59     int cl_cap;
60     ze_command_list_handle_t *cl_pool;  /* command list pool, one per device */
61     int cl_pool_size;
62     int cl_pool_cnt;
63     pthread_mutex_t cl_mutex;
64     ze_command_queue_group_properties_t *queueProperties;
65     int numQueueGroups;
66     pthread_mutex_t mutex;
67     int dev_id;
68 } yaksuri_zei_device_state_s;
69 
70 typedef struct {
71     ze_driver_handle_t driver;
72     uint32_t ndevices;
73     ze_device_handle_t *device;
74     uint32_t nsubdevices;
75     ze_device_handle_t **subdevices;
76     ze_context_handle_t context;
77 
78     bool **p2p;
79     pthread_mutex_t ze_mutex;
80 
81     int ev_pool_cap;
82     int throttle_threshold;
83 
84     yaksuri_zei_device_state_s *device_states;
85 } yaksuri_zei_global_s;
86 extern yaksuri_zei_global_s yaksuri_zei_global;
87 
88 /* Define abstraction for ze event and buffer command list with this */
89 typedef struct {
90     ze_event_handle_t ze_event;
91     ze_command_list_handle_t cl;
92     int dev_id;
93     int idx;
94 } yaksuri_zei_event_s;
95 
96 typedef struct yaksuri_zei_type_s {
97     int pack;
98     int unpack;
99     ze_kernel_handle_t *pack_kernels;
100     ze_kernel_handle_t *unpack_kernels;
101     yaksuri_zei_md_s **md;
102     pthread_mutex_t mdmutex;
103     uintptr_t num_elements;
104 } yaksuri_zei_type_s;
105 
106 typedef struct {
107     int yaksa_ze_use_copy_engine;
108     uintptr_t iov_pack_threshold;
109     uintptr_t iov_unpack_threshold;
110     struct {
111         bool is_valid;
112         struct {
113             ze_memory_allocation_properties_t prop;
114             ze_device_handle_t device;
115         } attr;
116     } inbuf, outbuf;
117 } yaksuri_zei_info_s;
118 
119 extern ze_module_handle_t *yaksuri_ze_modules[];
120 extern ze_kernel_handle_t *yaksuri_ze_kernels[];
121 
122 int yaksuri_zei_finalize_hook(void);
123 int yaksuri_zei_type_create_hook(yaksi_type_s * type);
124 int yaksuri_zei_type_free_hook(yaksi_type_s * type);
125 int yaksuri_zei_info_create_hook(yaksi_info_s * info);
126 int yaksuri_zei_info_free_hook(yaksi_info_s * info);
127 int yaksuri_zei_info_keyval_append(yaksi_info_s * info, const char *key, const void *val,
128                                    unsigned int vallen);
129 
130 int yaksuri_zei_event_record(int device, void **event);
131 int yaksuri_zei_event_query(void *event, int *completed);
132 int yaksuri_zei_add_dependency(int device1, int device2);
133 
134 int yaksuri_zei_get_ptr_attr(const void *inbuf, void *outbuf, yaksi_info_s * info,
135                              yaksur_ptr_attr_s * inattr, yaksur_ptr_attr_s * outattr);
136 
137 int yaksuri_zei_md_alloc(yaksi_type_s * type, int dev_id);
138 int yaksuri_zei_populate_pupfns(yaksi_type_s * type);
139 
140 int yaksuri_zei_ipack(const void *inbuf, void *outbuf, uintptr_t count, yaksi_type_s * type,
141                       yaksi_info_s * info, int target);
142 int yaksuri_zei_iunpack(const void *inbuf, void *outbuf, uintptr_t count, yaksi_type_s * type,
143                         yaksi_info_s * info, int target);
144 int yaksuri_zei_pup_is_supported(yaksi_type_s * type, bool * is_supported);
145 uintptr_t yaksuri_zei_get_iov_pack_threshold(yaksi_info_s * info);
146 uintptr_t yaksuri_zei_get_iov_unpack_threshold(yaksi_info_s * info);
147 
148 ze_result_t yaksuri_ze_init_module_kernel(void);
149 ze_result_t yaksuri_ze_finalize_module_kernel(void);
150 
151 int create_ze_event(int dev_id, ze_event_handle_t * ze_event, int *idx);
152 void recycle_command_list(ze_command_list_handle_t cl, int dev_id);
153 
154 /* *INDENT-OFF* */
155 #ifdef __cplusplus
156 }
157 #endif
158 /* *INDENT-ON* */
159 
160 #endif /* YAKSURI_ZEI_H_INCLUDED */
161