1 /*- 2 * Copyright (c) 2009-2012,2016 Microsoft Corp. 3 * Copyright (c) 2012 NetApp Inc. 4 * Copyright (c) 2012 Citrix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/lock.h> 34 #include <sys/mutex.h> 35 #include <sys/sysctl.h> 36 37 #include <dev/hyperv/vmbus/vmbus_reg.h> 38 #include <dev/hyperv/vmbus/vmbus_brvar.h> 39 40 /* Amount of space available for write */ 41 #define VMBUS_BR_WAVAIL(r, w, z) \ 42 (((w) >= (r)) ? ((z) - ((w) - (r))) : ((r) - (w))) 43 44 /* Increase bufing index */ 45 #define VMBUS_BR_IDXINC(idx, inc, sz) (((idx) + (inc)) % (sz)) 46 47 static int vmbus_br_sysctl_state(SYSCTL_HANDLER_ARGS); 48 static int vmbus_br_sysctl_state_bin(SYSCTL_HANDLER_ARGS); 49 static void vmbus_br_setup(struct vmbus_br *, void *, int); 50 51 static int 52 vmbus_br_sysctl_state(SYSCTL_HANDLER_ARGS) 53 { 54 const struct vmbus_br *br = arg1; 55 uint32_t rindex, windex, imask, ravail, wavail; 56 char state[256]; 57 58 rindex = br->vbr_rindex; 59 windex = br->vbr_windex; 60 imask = br->vbr_imask; 61 wavail = VMBUS_BR_WAVAIL(rindex, windex, br->vbr_dsize); 62 ravail = br->vbr_dsize - wavail; 63 64 snprintf(state, sizeof(state), 65 "rindex:%u windex:%u imask:%u ravail:%u wavail:%u", 66 rindex, windex, imask, ravail, wavail); 67 return sysctl_handle_string(oidp, state, sizeof(state), req); 68 } 69 70 /* 71 * Binary bufring states. 72 */ 73 static int 74 vmbus_br_sysctl_state_bin(SYSCTL_HANDLER_ARGS) 75 { 76 #define BR_STATE_RIDX 0 77 #define BR_STATE_WIDX 1 78 #define BR_STATE_IMSK 2 79 #define BR_STATE_RSPC 3 80 #define BR_STATE_WSPC 4 81 #define BR_STATE_MAX 5 82 83 const struct vmbus_br *br = arg1; 84 uint32_t rindex, windex, wavail, state[BR_STATE_MAX]; 85 86 rindex = br->vbr_rindex; 87 windex = br->vbr_windex; 88 wavail = VMBUS_BR_WAVAIL(rindex, windex, br->vbr_dsize); 89 90 state[BR_STATE_RIDX] = rindex; 91 state[BR_STATE_WIDX] = windex; 92 state[BR_STATE_IMSK] = br->vbr_imask; 93 state[BR_STATE_WSPC] = wavail; 94 state[BR_STATE_RSPC] = br->vbr_dsize - wavail; 95 96 return sysctl_handle_opaque(oidp, state, sizeof(state), req); 97 } 98 99 void 100 vmbus_br_sysctl_create(struct sysctl_ctx_list *ctx, struct sysctl_oid *br_tree, 101 struct vmbus_br *br, const char *name) 102 { 103 struct sysctl_oid *tree; 104 char desc[64]; 105 106 tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(br_tree), OID_AUTO, 107 name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 108 if (tree == NULL) 109 return; 110 111 snprintf(desc, sizeof(desc), "%s state", name); 112 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "state", 113 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 114 br, 0, vmbus_br_sysctl_state, "A", desc); 115 116 snprintf(desc, sizeof(desc), "%s binary state", name); 117 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "state_bin", 118 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 119 br, 0, vmbus_br_sysctl_state_bin, "IU", desc); 120 } 121 122 void 123 vmbus_rxbr_intr_mask(struct vmbus_rxbr *rbr) 124 { 125 rbr->rxbr_imask = 1; 126 mb(); 127 } 128 129 static __inline uint32_t 130 vmbus_rxbr_avail(const struct vmbus_rxbr *rbr) 131 { 132 uint32_t rindex, windex; 133 134 /* Get snapshot */ 135 rindex = rbr->rxbr_rindex; 136 windex = rbr->rxbr_windex; 137 138 return (rbr->rxbr_dsize - 139 VMBUS_BR_WAVAIL(rindex, windex, rbr->rxbr_dsize)); 140 } 141 142 uint32_t 143 vmbus_rxbr_intr_unmask(struct vmbus_rxbr *rbr) 144 { 145 rbr->rxbr_imask = 0; 146 mb(); 147 148 /* 149 * Now check to see if the ring buffer is still empty. 150 * If it is not, we raced and we need to process new 151 * incoming channel packets. 152 */ 153 return vmbus_rxbr_avail(rbr); 154 } 155 156 static void 157 vmbus_br_setup(struct vmbus_br *br, void *buf, int blen) 158 { 159 br->vbr = buf; 160 br->vbr_dsize = blen - sizeof(struct vmbus_bufring); 161 } 162 163 void 164 vmbus_rxbr_init(struct vmbus_rxbr *rbr) 165 { 166 mtx_init(&rbr->rxbr_lock, "vmbus_rxbr", NULL, MTX_SPIN); 167 } 168 169 void 170 vmbus_rxbr_deinit(struct vmbus_rxbr *rbr) 171 { 172 mtx_destroy(&rbr->rxbr_lock); 173 } 174 175 void 176 vmbus_rxbr_setup(struct vmbus_rxbr *rbr, void *buf, int blen) 177 { 178 vmbus_br_setup(&rbr->rxbr, buf, blen); 179 } 180 181 void 182 vmbus_txbr_init(struct vmbus_txbr *tbr) 183 { 184 mtx_init(&tbr->txbr_lock, "vmbus_txbr", NULL, MTX_SPIN); 185 } 186 187 void 188 vmbus_txbr_deinit(struct vmbus_txbr *tbr) 189 { 190 mtx_destroy(&tbr->txbr_lock); 191 } 192 193 void 194 vmbus_txbr_setup(struct vmbus_txbr *tbr, void *buf, int blen) 195 { 196 vmbus_br_setup(&tbr->txbr, buf, blen); 197 } 198 199 /* 200 * When we write to the ring buffer, check if the host needs to be 201 * signaled. 202 * 203 * The contract: 204 * - The host guarantees that while it is draining the TX bufring, 205 * it will set the br_imask to indicate it does not need to be 206 * interrupted when new data are added. 207 * - The host guarantees that it will completely drain the TX bufring 208 * before exiting the read loop. Further, once the TX bufring is 209 * empty, it will clear the br_imask and re-check to see if new 210 * data have arrived. 211 */ 212 static __inline boolean_t 213 vmbus_txbr_need_signal(const struct vmbus_txbr *tbr, uint32_t old_windex) 214 { 215 mb(); 216 if (tbr->txbr_imask) 217 return (FALSE); 218 219 __compiler_membar(); 220 221 /* 222 * This is the only case we need to signal when the 223 * ring transitions from being empty to non-empty. 224 */ 225 if (old_windex == tbr->txbr_rindex) 226 return (TRUE); 227 228 return (FALSE); 229 } 230 231 static __inline uint32_t 232 vmbus_txbr_avail(const struct vmbus_txbr *tbr) 233 { 234 uint32_t rindex, windex; 235 236 /* Get snapshot */ 237 rindex = tbr->txbr_rindex; 238 windex = tbr->txbr_windex; 239 240 return VMBUS_BR_WAVAIL(rindex, windex, tbr->txbr_dsize); 241 } 242 243 static __inline uint32_t 244 vmbus_txbr_copyto(const struct vmbus_txbr *tbr, uint32_t windex, 245 const void *src0, uint32_t cplen) 246 { 247 const uint8_t *src = src0; 248 uint8_t *br_data = tbr->txbr_data; 249 uint32_t br_dsize = tbr->txbr_dsize; 250 251 if (cplen > br_dsize - windex) { 252 uint32_t fraglen = br_dsize - windex; 253 254 /* Wrap-around detected */ 255 memcpy(br_data + windex, src, fraglen); 256 memcpy(br_data, src + fraglen, cplen - fraglen); 257 } else { 258 memcpy(br_data + windex, src, cplen); 259 } 260 return VMBUS_BR_IDXINC(windex, cplen, br_dsize); 261 } 262 263 /* 264 * Write scattered channel packet to TX bufring. 265 * 266 * The offset of this channel packet is written as a 64bits value 267 * immediately after this channel packet. 268 */ 269 int 270 vmbus_txbr_write(struct vmbus_txbr *tbr, const struct iovec iov[], int iovlen, 271 boolean_t *need_sig) 272 { 273 uint32_t old_windex, windex, total; 274 uint64_t save_windex; 275 int i; 276 277 total = 0; 278 for (i = 0; i < iovlen; i++) 279 total += iov[i].iov_len; 280 total += sizeof(save_windex); 281 282 mtx_lock_spin(&tbr->txbr_lock); 283 284 /* 285 * NOTE: 286 * If this write is going to make br_windex same as br_rindex, 287 * i.e. the available space for write is same as the write size, 288 * we can't do it then, since br_windex == br_rindex means that 289 * the bufring is empty. 290 */ 291 if (vmbus_txbr_avail(tbr) <= total) { 292 mtx_unlock_spin(&tbr->txbr_lock); 293 return (EAGAIN); 294 } 295 296 /* Save br_windex for later use */ 297 old_windex = tbr->txbr_windex; 298 299 /* 300 * Copy the scattered channel packet to the TX bufring. 301 */ 302 windex = old_windex; 303 for (i = 0; i < iovlen; i++) { 304 windex = vmbus_txbr_copyto(tbr, windex, 305 iov[i].iov_base, iov[i].iov_len); 306 } 307 308 /* 309 * Set the offset of the current channel packet. 310 */ 311 save_windex = ((uint64_t)old_windex) << 32; 312 windex = vmbus_txbr_copyto(tbr, windex, &save_windex, 313 sizeof(save_windex)); 314 315 /* 316 * Update the write index _after_ the channel packet 317 * is copied. 318 */ 319 __compiler_membar(); 320 tbr->txbr_windex = windex; 321 322 mtx_unlock_spin(&tbr->txbr_lock); 323 324 *need_sig = vmbus_txbr_need_signal(tbr, old_windex); 325 326 return (0); 327 } 328 329 static __inline uint32_t 330 vmbus_rxbr_copyfrom(const struct vmbus_rxbr *rbr, uint32_t rindex, 331 void *dst0, int cplen) 332 { 333 uint8_t *dst = dst0; 334 const uint8_t *br_data = rbr->rxbr_data; 335 uint32_t br_dsize = rbr->rxbr_dsize; 336 337 if (cplen > br_dsize - rindex) { 338 uint32_t fraglen = br_dsize - rindex; 339 340 /* Wrap-around detected. */ 341 memcpy(dst, br_data + rindex, fraglen); 342 memcpy(dst + fraglen, br_data, cplen - fraglen); 343 } else { 344 memcpy(dst, br_data + rindex, cplen); 345 } 346 return VMBUS_BR_IDXINC(rindex, cplen, br_dsize); 347 } 348 349 int 350 vmbus_rxbr_peek(struct vmbus_rxbr *rbr, void *data, int dlen) 351 { 352 mtx_lock_spin(&rbr->rxbr_lock); 353 354 /* 355 * The requested data and the 64bits channel packet 356 * offset should be there at least. 357 */ 358 if (vmbus_rxbr_avail(rbr) < dlen + sizeof(uint64_t)) { 359 mtx_unlock_spin(&rbr->rxbr_lock); 360 return (EAGAIN); 361 } 362 vmbus_rxbr_copyfrom(rbr, rbr->rxbr_rindex, data, dlen); 363 364 mtx_unlock_spin(&rbr->rxbr_lock); 365 366 return (0); 367 } 368 369 /* 370 * NOTE: 371 * We assume (dlen + skip) == sizeof(channel packet). 372 */ 373 int 374 vmbus_rxbr_read(struct vmbus_rxbr *rbr, void *data, int dlen, uint32_t skip) 375 { 376 uint32_t rindex, br_dsize = rbr->rxbr_dsize; 377 378 KASSERT(dlen + skip > 0, ("invalid dlen %d, offset %u", dlen, skip)); 379 380 mtx_lock_spin(&rbr->rxbr_lock); 381 382 if (vmbus_rxbr_avail(rbr) < dlen + skip + sizeof(uint64_t)) { 383 mtx_unlock_spin(&rbr->rxbr_lock); 384 return (EAGAIN); 385 } 386 387 /* 388 * Copy channel packet from RX bufring. 389 */ 390 rindex = VMBUS_BR_IDXINC(rbr->rxbr_rindex, skip, br_dsize); 391 rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen); 392 393 /* 394 * Discard this channel packet's 64bits offset, which is useless to us. 395 */ 396 rindex = VMBUS_BR_IDXINC(rindex, sizeof(uint64_t), br_dsize); 397 398 /* 399 * Update the read index _after_ the channel packet is fetched. 400 */ 401 __compiler_membar(); 402 rbr->rxbr_rindex = rindex; 403 404 mtx_unlock_spin(&rbr->rxbr_lock); 405 406 return (0); 407 } 408