1 /* 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2016 Alexander Motin <mav@FreeBSD.org> 5 * Copyright (c) 2015 Peter Grehan <grehan@freebsd.org> 6 * Copyright (c) 2013 Jeremiah Lott, Avere Systems 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer 14 * in this position and unchanged. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/types.h> 36 #ifndef WITHOUT_CAPSICUM 37 #include <sys/capsicum.h> 38 #endif 39 #include <sys/limits.h> 40 #include <sys/ioctl.h> 41 #include <sys/uio.h> 42 #include <net/ethernet.h> 43 #include <netinet/in.h> 44 #include <netinet/tcp.h> 45 46 #include <err.h> 47 #include <errno.h> 48 #include <fcntl.h> 49 #include <md5.h> 50 #include <stdio.h> 51 #include <stdlib.h> 52 #include <string.h> 53 #include <sysexits.h> 54 #include <unistd.h> 55 #include <pthread.h> 56 #include <pthread_np.h> 57 58 #include "e1000_regs.h" 59 #include "e1000_defines.h" 60 #include "mii.h" 61 62 #include "bhyverun.h" 63 #include "pci_emul.h" 64 #include "mevent.h" 65 66 /* Hardware/register definitions XXX: move some to common code. */ 67 #define E82545_VENDOR_ID_INTEL 0x8086 68 #define E82545_DEV_ID_82545EM_COPPER 0x100F 69 #define E82545_SUBDEV_ID 0x1008 70 71 #define E82545_REVISION_4 4 72 73 #define E82545_MDIC_DATA_MASK 0x0000FFFF 74 #define E82545_MDIC_OP_MASK 0x0c000000 75 #define E82545_MDIC_IE 0x20000000 76 77 #define E82545_EECD_FWE_DIS 0x00000010 /* Flash writes disabled */ 78 #define E82545_EECD_FWE_EN 0x00000020 /* Flash writes enabled */ 79 #define E82545_EECD_FWE_MASK 0x00000030 /* Flash writes mask */ 80 81 #define E82545_BAR_REGISTER 0 82 #define E82545_BAR_REGISTER_LEN (128*1024) 83 #define E82545_BAR_FLASH 1 84 #define E82545_BAR_FLASH_LEN (64*1024) 85 #define E82545_BAR_IO 2 86 #define E82545_BAR_IO_LEN 8 87 88 #define E82545_IOADDR 0x00000000 89 #define E82545_IODATA 0x00000004 90 #define E82545_IO_REGISTER_MAX 0x0001FFFF 91 #define E82545_IO_FLASH_BASE 0x00080000 92 #define E82545_IO_FLASH_MAX 0x000FFFFF 93 94 #define E82545_ARRAY_ENTRY(reg, offset) (reg + (offset<<2)) 95 #define E82545_RAR_MAX 15 96 #define E82545_MTA_MAX 127 97 #define E82545_VFTA_MAX 127 98 99 /* Slightly modified from the driver versions, hardcoded for 3 opcode bits, 100 * followed by 6 address bits. 101 * TODO: make opcode bits and addr bits configurable? 102 * NVM Commands - Microwire */ 103 #define E82545_NVM_OPCODE_BITS 3 104 #define E82545_NVM_ADDR_BITS 6 105 #define E82545_NVM_DATA_BITS 16 106 #define E82545_NVM_OPADDR_BITS (E82545_NVM_OPCODE_BITS + E82545_NVM_ADDR_BITS) 107 #define E82545_NVM_ADDR_MASK ((1 << E82545_NVM_ADDR_BITS)-1) 108 #define E82545_NVM_OPCODE_MASK \ 109 (((1 << E82545_NVM_OPCODE_BITS) - 1) << E82545_NVM_ADDR_BITS) 110 #define E82545_NVM_OPCODE_READ (0x6 << E82545_NVM_ADDR_BITS) /* read */ 111 #define E82545_NVM_OPCODE_WRITE (0x5 << E82545_NVM_ADDR_BITS) /* write */ 112 #define E82545_NVM_OPCODE_ERASE (0x7 << E82545_NVM_ADDR_BITS) /* erase */ 113 #define E82545_NVM_OPCODE_EWEN (0x4 << E82545_NVM_ADDR_BITS) /* wr-enable */ 114 115 #define E82545_NVM_EEPROM_SIZE 64 /* 64 * 16-bit values == 128K */ 116 117 #define E1000_ICR_SRPD 0x00010000 118 119 /* This is an arbitrary number. There is no hard limit on the chip. */ 120 #define I82545_MAX_TXSEGS 64 121 122 /* Legacy receive descriptor */ 123 struct e1000_rx_desc { 124 uint64_t buffer_addr; /* Address of the descriptor's data buffer */ 125 uint16_t length; /* Length of data DMAed into data buffer */ 126 uint16_t csum; /* Packet checksum */ 127 uint8_t status; /* Descriptor status */ 128 uint8_t errors; /* Descriptor Errors */ 129 uint16_t special; 130 }; 131 132 /* Transmit descriptor types */ 133 #define E1000_TXD_MASK (E1000_TXD_CMD_DEXT | 0x00F00000) 134 #define E1000_TXD_TYP_L (0) 135 #define E1000_TXD_TYP_C (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_C) 136 #define E1000_TXD_TYP_D (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D) 137 138 /* Legacy transmit descriptor */ 139 struct e1000_tx_desc { 140 uint64_t buffer_addr; /* Address of the descriptor's data buffer */ 141 union { 142 uint32_t data; 143 struct { 144 uint16_t length; /* Data buffer length */ 145 uint8_t cso; /* Checksum offset */ 146 uint8_t cmd; /* Descriptor control */ 147 } flags; 148 } lower; 149 union { 150 uint32_t data; 151 struct { 152 uint8_t status; /* Descriptor status */ 153 uint8_t css; /* Checksum start */ 154 uint16_t special; 155 } fields; 156 } upper; 157 }; 158 159 /* Context descriptor */ 160 struct e1000_context_desc { 161 union { 162 uint32_t ip_config; 163 struct { 164 uint8_t ipcss; /* IP checksum start */ 165 uint8_t ipcso; /* IP checksum offset */ 166 uint16_t ipcse; /* IP checksum end */ 167 } ip_fields; 168 } lower_setup; 169 union { 170 uint32_t tcp_config; 171 struct { 172 uint8_t tucss; /* TCP checksum start */ 173 uint8_t tucso; /* TCP checksum offset */ 174 uint16_t tucse; /* TCP checksum end */ 175 } tcp_fields; 176 } upper_setup; 177 uint32_t cmd_and_length; 178 union { 179 uint32_t data; 180 struct { 181 uint8_t status; /* Descriptor status */ 182 uint8_t hdr_len; /* Header length */ 183 uint16_t mss; /* Maximum segment size */ 184 } fields; 185 } tcp_seg_setup; 186 }; 187 188 /* Data descriptor */ 189 struct e1000_data_desc { 190 uint64_t buffer_addr; /* Address of the descriptor's buffer address */ 191 union { 192 uint32_t data; 193 struct { 194 uint16_t length; /* Data buffer length */ 195 uint8_t typ_len_ext; 196 uint8_t cmd; 197 } flags; 198 } lower; 199 union { 200 uint32_t data; 201 struct { 202 uint8_t status; /* Descriptor status */ 203 uint8_t popts; /* Packet Options */ 204 uint16_t special; 205 } fields; 206 } upper; 207 }; 208 209 union e1000_tx_udesc { 210 struct e1000_tx_desc td; 211 struct e1000_context_desc cd; 212 struct e1000_data_desc dd; 213 }; 214 215 /* Tx checksum info for a packet. */ 216 struct ck_info { 217 int ck_valid; /* ck_info is valid */ 218 uint8_t ck_start; /* start byte of cksum calcuation */ 219 uint8_t ck_off; /* offset of cksum insertion */ 220 uint16_t ck_len; /* length of cksum calc: 0 is to packet-end */ 221 }; 222 223 /* 224 * Debug printf 225 */ 226 static int e82545_debug = 0; 227 #define DPRINTF(msg,params...) if (e82545_debug) fprintf(stderr, "e82545: " msg, params) 228 #define WPRINTF(msg,params...) fprintf(stderr, "e82545: " msg, params) 229 230 #define MIN(a,b) (((a)<(b))?(a):(b)) 231 #define MAX(a,b) (((a)>(b))?(a):(b)) 232 233 /* s/w representation of the RAL/RAH regs */ 234 struct eth_uni { 235 int eu_valid; 236 int eu_addrsel; 237 struct ether_addr eu_eth; 238 }; 239 240 241 struct e82545_softc { 242 struct pci_devinst *esc_pi; 243 struct vmctx *esc_ctx; 244 struct mevent *esc_mevp; 245 struct mevent *esc_mevpitr; 246 pthread_mutex_t esc_mtx; 247 struct ether_addr esc_mac; 248 int esc_tapfd; 249 250 /* General */ 251 uint32_t esc_CTRL; /* x0000 device ctl */ 252 uint32_t esc_FCAL; /* x0028 flow ctl addr lo */ 253 uint32_t esc_FCAH; /* x002C flow ctl addr hi */ 254 uint32_t esc_FCT; /* x0030 flow ctl type */ 255 uint32_t esc_VET; /* x0038 VLAN eth type */ 256 uint32_t esc_FCTTV; /* x0170 flow ctl tx timer */ 257 uint32_t esc_LEDCTL; /* x0E00 LED control */ 258 uint32_t esc_PBA; /* x1000 pkt buffer allocation */ 259 260 /* Interrupt control */ 261 int esc_irq_asserted; 262 uint32_t esc_ICR; /* x00C0 cause read/clear */ 263 uint32_t esc_ITR; /* x00C4 intr throttling */ 264 uint32_t esc_ICS; /* x00C8 cause set */ 265 uint32_t esc_IMS; /* x00D0 mask set/read */ 266 uint32_t esc_IMC; /* x00D8 mask clear */ 267 268 /* Transmit */ 269 union e1000_tx_udesc *esc_txdesc; 270 struct e1000_context_desc esc_txctx; 271 pthread_t esc_tx_tid; 272 pthread_cond_t esc_tx_cond; 273 int esc_tx_enabled; 274 int esc_tx_active; 275 uint32_t esc_TXCW; /* x0178 transmit config */ 276 uint32_t esc_TCTL; /* x0400 transmit ctl */ 277 uint32_t esc_TIPG; /* x0410 inter-packet gap */ 278 uint16_t esc_AIT; /* x0458 Adaptive Interframe Throttle */ 279 uint64_t esc_tdba; /* verified 64-bit desc table addr */ 280 uint32_t esc_TDBAL; /* x3800 desc table addr, low bits */ 281 uint32_t esc_TDBAH; /* x3804 desc table addr, hi 32-bits */ 282 uint32_t esc_TDLEN; /* x3808 # descriptors in bytes */ 283 uint16_t esc_TDH; /* x3810 desc table head idx */ 284 uint16_t esc_TDHr; /* internal read version of TDH */ 285 uint16_t esc_TDT; /* x3818 desc table tail idx */ 286 uint32_t esc_TIDV; /* x3820 intr delay */ 287 uint32_t esc_TXDCTL; /* x3828 desc control */ 288 uint32_t esc_TADV; /* x382C intr absolute delay */ 289 290 /* L2 frame acceptance */ 291 struct eth_uni esc_uni[16]; /* 16 x unicast MAC addresses */ 292 uint32_t esc_fmcast[128]; /* Multicast filter bit-match */ 293 uint32_t esc_fvlan[128]; /* VLAN 4096-bit filter */ 294 295 /* Receive */ 296 struct e1000_rx_desc *esc_rxdesc; 297 pthread_cond_t esc_rx_cond; 298 int esc_rx_enabled; 299 int esc_rx_active; 300 int esc_rx_loopback; 301 uint32_t esc_RCTL; /* x0100 receive ctl */ 302 uint32_t esc_FCRTL; /* x2160 flow cntl thresh, low */ 303 uint32_t esc_FCRTH; /* x2168 flow cntl thresh, hi */ 304 uint64_t esc_rdba; /* verified 64-bit desc table addr */ 305 uint32_t esc_RDBAL; /* x2800 desc table addr, low bits */ 306 uint32_t esc_RDBAH; /* x2804 desc table addr, hi 32-bits*/ 307 uint32_t esc_RDLEN; /* x2808 #descriptors */ 308 uint16_t esc_RDH; /* x2810 desc table head idx */ 309 uint16_t esc_RDT; /* x2818 desc table tail idx */ 310 uint32_t esc_RDTR; /* x2820 intr delay */ 311 uint32_t esc_RXDCTL; /* x2828 desc control */ 312 uint32_t esc_RADV; /* x282C intr absolute delay */ 313 uint32_t esc_RSRPD; /* x2C00 recv small packet detect */ 314 uint32_t esc_RXCSUM; /* x5000 receive cksum ctl */ 315 316 /* IO Port register access */ 317 uint32_t io_addr; 318 319 /* Shadow copy of MDIC */ 320 uint32_t mdi_control; 321 /* Shadow copy of EECD */ 322 uint32_t eeprom_control; 323 /* Latest NVM in/out */ 324 uint16_t nvm_data; 325 uint16_t nvm_opaddr; 326 /* stats */ 327 uint32_t missed_pkt_count; /* dropped for no room in rx queue */ 328 uint32_t pkt_rx_by_size[6]; 329 uint32_t pkt_tx_by_size[6]; 330 uint32_t good_pkt_rx_count; 331 uint32_t bcast_pkt_rx_count; 332 uint32_t mcast_pkt_rx_count; 333 uint32_t good_pkt_tx_count; 334 uint32_t bcast_pkt_tx_count; 335 uint32_t mcast_pkt_tx_count; 336 uint32_t oversize_rx_count; 337 uint32_t tso_tx_count; 338 uint64_t good_octets_rx; 339 uint64_t good_octets_tx; 340 uint64_t missed_octets; /* counts missed and oversized */ 341 342 uint8_t nvm_bits:6; /* number of bits remaining in/out */ 343 uint8_t nvm_mode:2; 344 #define E82545_NVM_MODE_OPADDR 0x0 345 #define E82545_NVM_MODE_DATAIN 0x1 346 #define E82545_NVM_MODE_DATAOUT 0x2 347 /* EEPROM data */ 348 uint16_t eeprom_data[E82545_NVM_EEPROM_SIZE]; 349 }; 350 351 static void e82545_reset(struct e82545_softc *sc, int dev); 352 static void e82545_rx_enable(struct e82545_softc *sc); 353 static void e82545_rx_disable(struct e82545_softc *sc); 354 static void e82545_tap_callback(int fd, enum ev_type type, void *param); 355 static void e82545_tx_start(struct e82545_softc *sc); 356 static void e82545_tx_enable(struct e82545_softc *sc); 357 static void e82545_tx_disable(struct e82545_softc *sc); 358 359 static inline int 360 e82545_size_stat_index(uint32_t size) 361 { 362 if (size <= 64) { 363 return 0; 364 } else if (size >= 1024) { 365 return 5; 366 } else { 367 /* should be 1-4 */ 368 return (ffs(size) - 6); 369 } 370 } 371 372 static void 373 e82545_init_eeprom(struct e82545_softc *sc) 374 { 375 uint16_t checksum, i; 376 377 /* mac addr */ 378 sc->eeprom_data[NVM_MAC_ADDR] = ((uint16_t)sc->esc_mac.octet[0]) | 379 (((uint16_t)sc->esc_mac.octet[1]) << 8); 380 sc->eeprom_data[NVM_MAC_ADDR+1] = ((uint16_t)sc->esc_mac.octet[2]) | 381 (((uint16_t)sc->esc_mac.octet[3]) << 8); 382 sc->eeprom_data[NVM_MAC_ADDR+2] = ((uint16_t)sc->esc_mac.octet[4]) | 383 (((uint16_t)sc->esc_mac.octet[5]) << 8); 384 385 /* pci ids */ 386 sc->eeprom_data[NVM_SUB_DEV_ID] = E82545_SUBDEV_ID; 387 sc->eeprom_data[NVM_SUB_VEN_ID] = E82545_VENDOR_ID_INTEL; 388 sc->eeprom_data[NVM_DEV_ID] = E82545_DEV_ID_82545EM_COPPER; 389 sc->eeprom_data[NVM_VEN_ID] = E82545_VENDOR_ID_INTEL; 390 391 /* fill in the checksum */ 392 checksum = 0; 393 for (i = 0; i < NVM_CHECKSUM_REG; i++) { 394 checksum += sc->eeprom_data[i]; 395 } 396 checksum = NVM_SUM - checksum; 397 sc->eeprom_data[NVM_CHECKSUM_REG] = checksum; 398 DPRINTF("eeprom checksum: 0x%x\r\n", checksum); 399 } 400 401 static void 402 e82545_write_mdi(struct e82545_softc *sc, uint8_t reg_addr, 403 uint8_t phy_addr, uint32_t data) 404 { 405 DPRINTF("Write mdi reg:0x%x phy:0x%x data: 0x%x\r\n", reg_addr, phy_addr, data); 406 } 407 408 static uint32_t 409 e82545_read_mdi(struct e82545_softc *sc, uint8_t reg_addr, 410 uint8_t phy_addr) 411 { 412 //DPRINTF("Read mdi reg:0x%x phy:0x%x\r\n", reg_addr, phy_addr); 413 switch (reg_addr) { 414 case PHY_STATUS: 415 return (MII_SR_LINK_STATUS | MII_SR_AUTONEG_CAPS | 416 MII_SR_AUTONEG_COMPLETE); 417 case PHY_AUTONEG_ADV: 418 return NWAY_AR_SELECTOR_FIELD; 419 case PHY_LP_ABILITY: 420 return 0; 421 case PHY_1000T_STATUS: 422 return (SR_1000T_LP_FD_CAPS | SR_1000T_REMOTE_RX_STATUS | 423 SR_1000T_LOCAL_RX_STATUS); 424 case PHY_ID1: 425 return (M88E1011_I_PHY_ID >> 16) & 0xFFFF; 426 case PHY_ID2: 427 return (M88E1011_I_PHY_ID | E82545_REVISION_4) & 0xFFFF; 428 default: 429 DPRINTF("Unknown mdi read reg:0x%x phy:0x%x\r\n", reg_addr, phy_addr); 430 return 0; 431 } 432 /* not reached */ 433 } 434 435 static void 436 e82545_eecd_strobe(struct e82545_softc *sc) 437 { 438 /* Microwire state machine */ 439 /* 440 DPRINTF("eeprom state machine srtobe " 441 "0x%x 0x%x 0x%x 0x%x\r\n", 442 sc->nvm_mode, sc->nvm_bits, 443 sc->nvm_opaddr, sc->nvm_data);*/ 444 445 if (sc->nvm_bits == 0) { 446 DPRINTF("eeprom state machine not expecting data! " 447 "0x%x 0x%x 0x%x 0x%x\r\n", 448 sc->nvm_mode, sc->nvm_bits, 449 sc->nvm_opaddr, sc->nvm_data); 450 return; 451 } 452 sc->nvm_bits--; 453 if (sc->nvm_mode == E82545_NVM_MODE_DATAOUT) { 454 /* shifting out */ 455 if (sc->nvm_data & 0x8000) { 456 sc->eeprom_control |= E1000_EECD_DO; 457 } else { 458 sc->eeprom_control &= ~E1000_EECD_DO; 459 } 460 sc->nvm_data <<= 1; 461 if (sc->nvm_bits == 0) { 462 /* read done, back to opcode mode. */ 463 sc->nvm_opaddr = 0; 464 sc->nvm_mode = E82545_NVM_MODE_OPADDR; 465 sc->nvm_bits = E82545_NVM_OPADDR_BITS; 466 } 467 } else if (sc->nvm_mode == E82545_NVM_MODE_DATAIN) { 468 /* shifting in */ 469 sc->nvm_data <<= 1; 470 if (sc->eeprom_control & E1000_EECD_DI) { 471 sc->nvm_data |= 1; 472 } 473 if (sc->nvm_bits == 0) { 474 /* eeprom write */ 475 uint16_t op = sc->nvm_opaddr & E82545_NVM_OPCODE_MASK; 476 uint16_t addr = sc->nvm_opaddr & E82545_NVM_ADDR_MASK; 477 if (op != E82545_NVM_OPCODE_WRITE) { 478 DPRINTF("Illegal eeprom write op 0x%x\r\n", 479 sc->nvm_opaddr); 480 } else if (addr >= E82545_NVM_EEPROM_SIZE) { 481 DPRINTF("Illegal eeprom write addr 0x%x\r\n", 482 sc->nvm_opaddr); 483 } else { 484 DPRINTF("eeprom write eeprom[0x%x] = 0x%x\r\n", 485 addr, sc->nvm_data); 486 sc->eeprom_data[addr] = sc->nvm_data; 487 } 488 /* back to opcode mode */ 489 sc->nvm_opaddr = 0; 490 sc->nvm_mode = E82545_NVM_MODE_OPADDR; 491 sc->nvm_bits = E82545_NVM_OPADDR_BITS; 492 } 493 } else if (sc->nvm_mode == E82545_NVM_MODE_OPADDR) { 494 sc->nvm_opaddr <<= 1; 495 if (sc->eeprom_control & E1000_EECD_DI) { 496 sc->nvm_opaddr |= 1; 497 } 498 if (sc->nvm_bits == 0) { 499 uint16_t op = sc->nvm_opaddr & E82545_NVM_OPCODE_MASK; 500 switch (op) { 501 case E82545_NVM_OPCODE_EWEN: 502 DPRINTF("eeprom write enable: 0x%x\r\n", 503 sc->nvm_opaddr); 504 /* back to opcode mode */ 505 sc->nvm_opaddr = 0; 506 sc->nvm_mode = E82545_NVM_MODE_OPADDR; 507 sc->nvm_bits = E82545_NVM_OPADDR_BITS; 508 break; 509 case E82545_NVM_OPCODE_READ: 510 { 511 uint16_t addr = sc->nvm_opaddr & 512 E82545_NVM_ADDR_MASK; 513 sc->nvm_mode = E82545_NVM_MODE_DATAOUT; 514 sc->nvm_bits = E82545_NVM_DATA_BITS; 515 if (addr < E82545_NVM_EEPROM_SIZE) { 516 sc->nvm_data = sc->eeprom_data[addr]; 517 DPRINTF("eeprom read: eeprom[0x%x] = 0x%x\r\n", 518 addr, sc->nvm_data); 519 } else { 520 DPRINTF("eeprom illegal read: 0x%x\r\n", 521 sc->nvm_opaddr); 522 sc->nvm_data = 0; 523 } 524 break; 525 } 526 case E82545_NVM_OPCODE_WRITE: 527 sc->nvm_mode = E82545_NVM_MODE_DATAIN; 528 sc->nvm_bits = E82545_NVM_DATA_BITS; 529 sc->nvm_data = 0; 530 break; 531 default: 532 DPRINTF("eeprom unknown op: 0x%x\r\r", 533 sc->nvm_opaddr); 534 /* back to opcode mode */ 535 sc->nvm_opaddr = 0; 536 sc->nvm_mode = E82545_NVM_MODE_OPADDR; 537 sc->nvm_bits = E82545_NVM_OPADDR_BITS; 538 } 539 } 540 } else { 541 DPRINTF("eeprom state machine wrong state! " 542 "0x%x 0x%x 0x%x 0x%x\r\n", 543 sc->nvm_mode, sc->nvm_bits, 544 sc->nvm_opaddr, sc->nvm_data); 545 } 546 } 547 548 static void 549 e82545_itr_callback(int fd, enum ev_type type, void *param) 550 { 551 uint32_t new; 552 struct e82545_softc *sc = param; 553 554 pthread_mutex_lock(&sc->esc_mtx); 555 new = sc->esc_ICR & sc->esc_IMS; 556 if (new && !sc->esc_irq_asserted) { 557 DPRINTF("itr callback: lintr assert %x\r\n", new); 558 sc->esc_irq_asserted = 1; 559 pci_lintr_assert(sc->esc_pi); 560 } else { 561 mevent_delete(sc->esc_mevpitr); 562 sc->esc_mevpitr = NULL; 563 } 564 pthread_mutex_unlock(&sc->esc_mtx); 565 } 566 567 static void 568 e82545_icr_assert(struct e82545_softc *sc, uint32_t bits) 569 { 570 uint32_t new; 571 572 DPRINTF("icr assert: 0x%x\r\n", bits); 573 574 /* 575 * An interrupt is only generated if bits are set that 576 * aren't already in the ICR, these bits are unmasked, 577 * and there isn't an interrupt already pending. 578 */ 579 new = bits & ~sc->esc_ICR & sc->esc_IMS; 580 sc->esc_ICR |= bits; 581 582 if (new == 0) { 583 DPRINTF("icr assert: masked %x, ims %x\r\n", new, sc->esc_IMS); 584 } else if (sc->esc_mevpitr != NULL) { 585 DPRINTF("icr assert: throttled %x, ims %x\r\n", new, sc->esc_IMS); 586 } else if (!sc->esc_irq_asserted) { 587 DPRINTF("icr assert: lintr assert %x\r\n", new); 588 sc->esc_irq_asserted = 1; 589 pci_lintr_assert(sc->esc_pi); 590 if (sc->esc_ITR != 0) { 591 sc->esc_mevpitr = mevent_add( 592 (sc->esc_ITR + 3905) / 3906, /* 256ns -> 1ms */ 593 EVF_TIMER, e82545_itr_callback, sc); 594 } 595 } 596 } 597 598 static void 599 e82545_ims_change(struct e82545_softc *sc, uint32_t bits) 600 { 601 uint32_t new; 602 603 /* 604 * Changing the mask may allow previously asserted 605 * but masked interrupt requests to generate an interrupt. 606 */ 607 new = bits & sc->esc_ICR & ~sc->esc_IMS; 608 sc->esc_IMS |= bits; 609 610 if (new == 0) { 611 DPRINTF("ims change: masked %x, ims %x\r\n", new, sc->esc_IMS); 612 } else if (sc->esc_mevpitr != NULL) { 613 DPRINTF("ims change: throttled %x, ims %x\r\n", new, sc->esc_IMS); 614 } else if (!sc->esc_irq_asserted) { 615 DPRINTF("ims change: lintr assert %x\n\r", new); 616 sc->esc_irq_asserted = 1; 617 pci_lintr_assert(sc->esc_pi); 618 if (sc->esc_ITR != 0) { 619 sc->esc_mevpitr = mevent_add( 620 (sc->esc_ITR + 3905) / 3906, /* 256ns -> 1ms */ 621 EVF_TIMER, e82545_itr_callback, sc); 622 } 623 } 624 } 625 626 static void 627 e82545_icr_deassert(struct e82545_softc *sc, uint32_t bits) 628 { 629 630 DPRINTF("icr deassert: 0x%x\r\n", bits); 631 sc->esc_ICR &= ~bits; 632 633 /* 634 * If there are no longer any interrupt sources and there 635 * was an asserted interrupt, clear it 636 */ 637 if (sc->esc_irq_asserted && !(sc->esc_ICR & sc->esc_IMS)) { 638 DPRINTF("icr deassert: lintr deassert %x\r\n", bits); 639 pci_lintr_deassert(sc->esc_pi); 640 sc->esc_irq_asserted = 0; 641 } 642 } 643 644 static void 645 e82545_intr_write(struct e82545_softc *sc, uint32_t offset, uint32_t value) 646 { 647 648 DPRINTF("intr_write: off %x, val %x\n\r", offset, value); 649 650 switch (offset) { 651 case E1000_ICR: 652 e82545_icr_deassert(sc, value); 653 break; 654 case E1000_ITR: 655 sc->esc_ITR = value; 656 break; 657 case E1000_ICS: 658 sc->esc_ICS = value; /* not used: store for debug */ 659 e82545_icr_assert(sc, value); 660 break; 661 case E1000_IMS: 662 e82545_ims_change(sc, value); 663 break; 664 case E1000_IMC: 665 sc->esc_IMC = value; /* for debug */ 666 sc->esc_IMS &= ~value; 667 // XXX clear interrupts if all ICR bits now masked 668 // and interrupt was pending ? 669 break; 670 default: 671 break; 672 } 673 } 674 675 static uint32_t 676 e82545_intr_read(struct e82545_softc *sc, uint32_t offset) 677 { 678 uint32_t retval; 679 680 retval = 0; 681 682 DPRINTF("intr_read: off %x\n\r", offset); 683 684 switch (offset) { 685 case E1000_ICR: 686 retval = sc->esc_ICR; 687 sc->esc_ICR = 0; 688 e82545_icr_deassert(sc, ~0); 689 break; 690 case E1000_ITR: 691 retval = sc->esc_ITR; 692 break; 693 case E1000_ICS: 694 /* write-only register */ 695 break; 696 case E1000_IMS: 697 retval = sc->esc_IMS; 698 break; 699 case E1000_IMC: 700 /* write-only register */ 701 break; 702 default: 703 break; 704 } 705 706 return (retval); 707 } 708 709 static void 710 e82545_devctl(struct e82545_softc *sc, uint32_t val) 711 { 712 713 sc->esc_CTRL = val & ~E1000_CTRL_RST; 714 715 if (val & E1000_CTRL_RST) { 716 DPRINTF("e1k: s/w reset, ctl %x\n", val); 717 e82545_reset(sc, 1); 718 } 719 /* XXX check for phy reset ? */ 720 } 721 722 static void 723 e82545_rx_update_rdba(struct e82545_softc *sc) 724 { 725 726 /* XXX verify desc base/len within phys mem range */ 727 sc->esc_rdba = (uint64_t)sc->esc_RDBAH << 32 | 728 sc->esc_RDBAL; 729 730 /* Cache host mapping of guest descriptor array */ 731 sc->esc_rxdesc = paddr_guest2host(sc->esc_ctx, 732 sc->esc_rdba, sc->esc_RDLEN); 733 } 734 735 static void 736 e82545_rx_ctl(struct e82545_softc *sc, uint32_t val) 737 { 738 int on; 739 740 on = ((val & E1000_RCTL_EN) == E1000_RCTL_EN); 741 742 /* Save RCTL after stripping reserved bits 31:27,24,21,14,11:10,0 */ 743 sc->esc_RCTL = val & ~0xF9204c01; 744 745 DPRINTF("rx_ctl - %s RCTL %x, val %x\n", 746 on ? "on" : "off", sc->esc_RCTL, val); 747 748 /* state change requested */ 749 if (on != sc->esc_rx_enabled) { 750 if (on) { 751 /* Catch disallowed/unimplemented settings */ 752 //assert(!(val & E1000_RCTL_LBM_TCVR)); 753 754 if (sc->esc_RCTL & E1000_RCTL_LBM_TCVR) { 755 sc->esc_rx_loopback = 1; 756 } else { 757 sc->esc_rx_loopback = 0; 758 } 759 760 e82545_rx_update_rdba(sc); 761 e82545_rx_enable(sc); 762 } else { 763 e82545_rx_disable(sc); 764 sc->esc_rx_loopback = 0; 765 sc->esc_rdba = 0; 766 sc->esc_rxdesc = NULL; 767 } 768 } 769 } 770 771 static void 772 e82545_tx_update_tdba(struct e82545_softc *sc) 773 { 774 775 /* XXX verify desc base/len within phys mem range */ 776 sc->esc_tdba = (uint64_t)sc->esc_TDBAH << 32 | sc->esc_TDBAL; 777 778 /* Cache host mapping of guest descriptor array */ 779 sc->esc_txdesc = paddr_guest2host(sc->esc_ctx, sc->esc_tdba, 780 sc->esc_TDLEN); 781 } 782 783 static void 784 e82545_tx_ctl(struct e82545_softc *sc, uint32_t val) 785 { 786 int on; 787 788 on = ((val & E1000_TCTL_EN) == E1000_TCTL_EN); 789 790 /* ignore TCTL_EN settings that don't change state */ 791 if (on == sc->esc_tx_enabled) 792 return; 793 794 if (on) { 795 e82545_tx_update_tdba(sc); 796 e82545_tx_enable(sc); 797 } else { 798 e82545_tx_disable(sc); 799 sc->esc_tdba = 0; 800 sc->esc_txdesc = NULL; 801 } 802 803 /* Save TCTL value after stripping reserved bits 31:25,23,2,0 */ 804 sc->esc_TCTL = val & ~0xFE800005; 805 } 806 807 int 808 e82545_bufsz(uint32_t rctl) 809 { 810 811 switch (rctl & (E1000_RCTL_BSEX | E1000_RCTL_SZ_256)) { 812 case (E1000_RCTL_SZ_2048): return (2048); 813 case (E1000_RCTL_SZ_1024): return (1024); 814 case (E1000_RCTL_SZ_512): return (512); 815 case (E1000_RCTL_SZ_256): return (256); 816 case (E1000_RCTL_BSEX|E1000_RCTL_SZ_16384): return (16384); 817 case (E1000_RCTL_BSEX|E1000_RCTL_SZ_8192): return (8192); 818 case (E1000_RCTL_BSEX|E1000_RCTL_SZ_4096): return (4096); 819 } 820 return (256); /* Forbidden value. */ 821 } 822 823 static uint8_t dummybuf[2048]; 824 825 /* XXX one packet at a time until this is debugged */ 826 static void 827 e82545_tap_callback(int fd, enum ev_type type, void *param) 828 { 829 struct e82545_softc *sc = param; 830 struct e1000_rx_desc *rxd; 831 struct iovec vec[64]; 832 int left, len, lim, maxpktsz, maxpktdesc, bufsz, i, n, size; 833 uint32_t cause = 0; 834 uint16_t *tp, tag, head; 835 836 pthread_mutex_lock(&sc->esc_mtx); 837 DPRINTF("rx_run: head %x, tail %x\r\n", sc->esc_RDH, sc->esc_RDT); 838 839 if (!sc->esc_rx_enabled || sc->esc_rx_loopback) { 840 DPRINTF("rx disabled (!%d || %d) -- packet(s) dropped\r\n", 841 sc->esc_rx_enabled, sc->esc_rx_loopback); 842 while (read(sc->esc_tapfd, dummybuf, sizeof(dummybuf)) > 0) { 843 } 844 goto done1; 845 } 846 bufsz = e82545_bufsz(sc->esc_RCTL); 847 maxpktsz = (sc->esc_RCTL & E1000_RCTL_LPE) ? 16384 : 1522; 848 maxpktdesc = (maxpktsz + bufsz - 1) / bufsz; 849 size = sc->esc_RDLEN / 16; 850 head = sc->esc_RDH; 851 left = (size + sc->esc_RDT - head) % size; 852 if (left < maxpktdesc) { 853 DPRINTF("rx overflow (%d < %d) -- packet(s) dropped\r\n", 854 left, maxpktdesc); 855 while (read(sc->esc_tapfd, dummybuf, sizeof(dummybuf)) > 0) { 856 } 857 goto done1; 858 } 859 860 sc->esc_rx_active = 1; 861 pthread_mutex_unlock(&sc->esc_mtx); 862 863 for (lim = size / 4; lim > 0 && left >= maxpktdesc; lim -= n) { 864 865 /* Grab rx descriptor pointed to by the head pointer */ 866 for (i = 0; i < maxpktdesc; i++) { 867 rxd = &sc->esc_rxdesc[(head + i) % size]; 868 vec[i].iov_base = paddr_guest2host(sc->esc_ctx, 869 rxd->buffer_addr, bufsz); 870 vec[i].iov_len = bufsz; 871 } 872 len = readv(sc->esc_tapfd, vec, maxpktdesc); 873 if (len <= 0) { 874 DPRINTF("tap: readv() returned %d\n", len); 875 goto done; 876 } 877 878 /* 879 * Adjust the packet length based on whether the CRC needs 880 * to be stripped or if the packet is less than the minimum 881 * eth packet size. 882 */ 883 if (len < ETHER_MIN_LEN - ETHER_CRC_LEN) 884 len = ETHER_MIN_LEN - ETHER_CRC_LEN; 885 if (!(sc->esc_RCTL & E1000_RCTL_SECRC)) 886 len += ETHER_CRC_LEN; 887 n = (len + bufsz - 1) / bufsz; 888 889 DPRINTF("packet read %d bytes, %d segs, head %d\r\n", 890 len, n, head); 891 892 /* Apply VLAN filter. */ 893 tp = (uint16_t *)vec[0].iov_base + 6; 894 if ((sc->esc_RCTL & E1000_RCTL_VFE) && 895 (ntohs(tp[0]) == sc->esc_VET)) { 896 tag = ntohs(tp[1]) & 0x0fff; 897 if ((sc->esc_fvlan[tag >> 5] & 898 (1 << (tag & 0x1f))) != 0) { 899 DPRINTF("known VLAN %d\r\n", tag); 900 } else { 901 DPRINTF("unknown VLAN %d\r\n", tag); 902 n = 0; 903 continue; 904 } 905 } 906 907 /* Update all consumed descriptors. */ 908 for (i = 0; i < n - 1; i++) { 909 rxd = &sc->esc_rxdesc[(head + i) % size]; 910 rxd->length = bufsz; 911 rxd->csum = 0; 912 rxd->errors = 0; 913 rxd->special = 0; 914 rxd->status = E1000_RXD_STAT_DD; 915 } 916 rxd = &sc->esc_rxdesc[(head + i) % size]; 917 rxd->length = len % bufsz; 918 rxd->csum = 0; 919 rxd->errors = 0; 920 rxd->special = 0; 921 /* XXX signal no checksum for now */ 922 rxd->status = E1000_RXD_STAT_PIF | E1000_RXD_STAT_IXSM | 923 E1000_RXD_STAT_EOP | E1000_RXD_STAT_DD; 924 925 /* Schedule receive interrupts. */ 926 if (len <= sc->esc_RSRPD) { 927 cause |= E1000_ICR_SRPD | E1000_ICR_RXT0; 928 } else { 929 /* XXX: RDRT and RADV timers should be here. */ 930 cause |= E1000_ICR_RXT0; 931 } 932 933 head = (head + n) % size; 934 left -= n; 935 } 936 937 done: 938 pthread_mutex_lock(&sc->esc_mtx); 939 sc->esc_rx_active = 0; 940 if (sc->esc_rx_enabled == 0) 941 pthread_cond_signal(&sc->esc_rx_cond); 942 943 sc->esc_RDH = head; 944 /* Respect E1000_RCTL_RDMTS */ 945 left = (size + sc->esc_RDT - head) % size; 946 if (left < (size >> (((sc->esc_RCTL >> 8) & 3) + 1))) 947 cause |= E1000_ICR_RXDMT0; 948 /* Assert all accumulated interrupts. */ 949 if (cause != 0) 950 e82545_icr_assert(sc, cause); 951 done1: 952 DPRINTF("rx_run done: head %x, tail %x\r\n", sc->esc_RDH, sc->esc_RDT); 953 pthread_mutex_unlock(&sc->esc_mtx); 954 } 955 956 static uint16_t 957 e82545_carry(uint32_t sum) 958 { 959 960 sum = (sum & 0xFFFF) + (sum >> 16); 961 if (sum > 0xFFFF) 962 sum -= 0xFFFF; 963 return (sum); 964 } 965 966 static uint16_t 967 e82545_buf_checksum(uint8_t *buf, int len) 968 { 969 int i; 970 uint32_t sum = 0; 971 972 /* Checksum all the pairs of bytes first... */ 973 for (i = 0; i < (len & ~1U); i += 2) 974 sum += *((u_int16_t *)(buf + i)); 975 976 /* 977 * If there's a single byte left over, checksum it, too. 978 * Network byte order is big-endian, so the remaining byte is 979 * the high byte. 980 */ 981 if (i < len) 982 sum += htons(buf[i] << 8); 983 984 return (e82545_carry(sum)); 985 } 986 987 static uint16_t 988 e82545_iov_checksum(struct iovec *iov, int iovcnt, int off, int len) 989 { 990 int now, odd; 991 uint32_t sum = 0, s; 992 993 /* Skip completely unneeded vectors. */ 994 while (iovcnt > 0 && iov->iov_len <= off && off > 0) { 995 off -= iov->iov_len; 996 iov++; 997 iovcnt--; 998 } 999 1000 /* Calculate checksum of requested range. */ 1001 odd = 0; 1002 while (len > 0 && iovcnt > 0) { 1003 now = MIN(len, iov->iov_len - off); 1004 s = e82545_buf_checksum(iov->iov_base + off, now); 1005 sum += odd ? (s << 8) : s; 1006 odd ^= (now & 1); 1007 len -= now; 1008 off = 0; 1009 iov++; 1010 iovcnt--; 1011 } 1012 1013 return (e82545_carry(sum)); 1014 } 1015 1016 /* 1017 * Return the transmit descriptor type. 1018 */ 1019 int 1020 e82545_txdesc_type(uint32_t lower) 1021 { 1022 int type; 1023 1024 type = 0; 1025 1026 if (lower & E1000_TXD_CMD_DEXT) 1027 type = lower & E1000_TXD_MASK; 1028 1029 return (type); 1030 } 1031 1032 static void 1033 e82545_transmit_checksum(struct iovec *iov, int iovcnt, struct ck_info *ck) 1034 { 1035 uint16_t cksum; 1036 int cklen; 1037 1038 DPRINTF("tx cksum: iovcnt/s/off/len %d/%d/%d/%d\r\n", 1039 iovcnt, ck->ck_start, ck->ck_off, ck->ck_len); 1040 cklen = ck->ck_len ? ck->ck_len - ck->ck_start + 1 : INT_MAX; 1041 cksum = e82545_iov_checksum(iov, iovcnt, ck->ck_start, cklen); 1042 *(uint16_t *)((uint8_t *)iov[0].iov_base + ck->ck_off) = ~cksum; 1043 } 1044 1045 static void 1046 e82545_transmit_backend(struct e82545_softc *sc, struct iovec *iov, int iovcnt) 1047 { 1048 1049 if (sc->esc_tapfd == -1) 1050 return; 1051 1052 (void) writev(sc->esc_tapfd, iov, iovcnt); 1053 } 1054 1055 static void 1056 e82545_transmit_done(struct e82545_softc *sc, uint16_t head, uint16_t tail, 1057 uint16_t dsize, int *tdwb) 1058 { 1059 union e1000_tx_udesc *dsc; 1060 1061 for ( ; head != tail; head = (head + 1) % dsize) { 1062 dsc = &sc->esc_txdesc[head]; 1063 if (dsc->td.lower.data & E1000_TXD_CMD_RS) { 1064 dsc->td.upper.data |= E1000_TXD_STAT_DD; 1065 *tdwb = 1; 1066 } 1067 } 1068 } 1069 1070 static int 1071 e82545_transmit(struct e82545_softc *sc, uint16_t head, uint16_t tail, 1072 uint16_t dsize, uint16_t *rhead, int *tdwb) 1073 { 1074 uint8_t *hdr, *hdrp; 1075 struct iovec iovb[I82545_MAX_TXSEGS + 2]; 1076 struct iovec tiov[I82545_MAX_TXSEGS + 2]; 1077 struct e1000_context_desc *cd; 1078 struct ck_info ckinfo[2]; 1079 struct iovec *iov; 1080 union e1000_tx_udesc *dsc; 1081 int desc, dtype, len, ntype, iovcnt, tlen, hdrlen, vlen, tcp, tso; 1082 int mss, paylen, seg, tiovcnt, left, now, nleft, nnow, pv, pvoff; 1083 uint32_t tcpsum, tcpseq; 1084 uint16_t ipcs, tcpcs, ipid, ohead; 1085 1086 ckinfo[0].ck_valid = ckinfo[1].ck_valid = 0; 1087 iovcnt = 0; 1088 tlen = 0; 1089 ntype = 0; 1090 tso = 0; 1091 ohead = head; 1092 1093 /* iovb[0/1] may be used for writable copy of headers. */ 1094 iov = &iovb[2]; 1095 1096 for (desc = 0; ; desc++, head = (head + 1) % dsize) { 1097 if (head == tail) { 1098 *rhead = head; 1099 return (0); 1100 } 1101 dsc = &sc->esc_txdesc[head]; 1102 dtype = e82545_txdesc_type(dsc->td.lower.data); 1103 1104 if (desc == 0) { 1105 switch (dtype) { 1106 case E1000_TXD_TYP_C: 1107 DPRINTF("tx ctxt desc idx %d: %016jx " 1108 "%08x%08x\r\n", 1109 head, dsc->td.buffer_addr, 1110 dsc->td.upper.data, dsc->td.lower.data); 1111 /* Save context and return */ 1112 sc->esc_txctx = dsc->cd; 1113 goto done; 1114 case E1000_TXD_TYP_L: 1115 DPRINTF("tx legacy desc idx %d: %08x%08x\r\n", 1116 head, dsc->td.upper.data, dsc->td.lower.data); 1117 /* 1118 * legacy cksum start valid in first descriptor 1119 */ 1120 ntype = dtype; 1121 ckinfo[0].ck_start = dsc->td.upper.fields.css; 1122 break; 1123 case E1000_TXD_TYP_D: 1124 DPRINTF("tx data desc idx %d: %08x%08x\r\n", 1125 head, dsc->td.upper.data, dsc->td.lower.data); 1126 ntype = dtype; 1127 break; 1128 default: 1129 break; 1130 } 1131 } else { 1132 /* Descriptor type must be consistent */ 1133 assert(dtype == ntype); 1134 DPRINTF("tx next desc idx %d: %08x%08x\r\n", 1135 head, dsc->td.upper.data, dsc->td.lower.data); 1136 } 1137 1138 len = (dtype == E1000_TXD_TYP_L) ? dsc->td.lower.flags.length : 1139 dsc->dd.lower.data & 0xFFFFF; 1140 1141 if (len > 0) { 1142 /* Strip checksum supplied by guest. */ 1143 if ((dsc->td.lower.data & E1000_TXD_CMD_EOP) != 0 && 1144 (dsc->td.lower.data & E1000_TXD_CMD_IFCS) == 0) 1145 len -= 2; 1146 tlen += len; 1147 if (iovcnt < I82545_MAX_TXSEGS) { 1148 iov[iovcnt].iov_base = paddr_guest2host( 1149 sc->esc_ctx, dsc->td.buffer_addr, len); 1150 iov[iovcnt].iov_len = len; 1151 } 1152 iovcnt++; 1153 } 1154 1155 /* 1156 * Pull out info that is valid in the final descriptor 1157 * and exit descriptor loop. 1158 */ 1159 if (dsc->td.lower.data & E1000_TXD_CMD_EOP) { 1160 if (dtype == E1000_TXD_TYP_L) { 1161 if (dsc->td.lower.data & E1000_TXD_CMD_IC) { 1162 ckinfo[0].ck_valid = 1; 1163 ckinfo[0].ck_off = 1164 dsc->td.lower.flags.cso; 1165 ckinfo[0].ck_len = 0; 1166 } 1167 } else { 1168 cd = &sc->esc_txctx; 1169 if (dsc->dd.lower.data & E1000_TXD_CMD_TSE) 1170 tso = 1; 1171 if (dsc->dd.upper.fields.popts & 1172 E1000_TXD_POPTS_IXSM) 1173 ckinfo[0].ck_valid = 1; 1174 if (dsc->dd.upper.fields.popts & 1175 E1000_TXD_POPTS_IXSM || tso) { 1176 ckinfo[0].ck_start = 1177 cd->lower_setup.ip_fields.ipcss; 1178 ckinfo[0].ck_off = 1179 cd->lower_setup.ip_fields.ipcso; 1180 ckinfo[0].ck_len = 1181 cd->lower_setup.ip_fields.ipcse; 1182 } 1183 if (dsc->dd.upper.fields.popts & 1184 E1000_TXD_POPTS_TXSM) 1185 ckinfo[1].ck_valid = 1; 1186 if (dsc->dd.upper.fields.popts & 1187 E1000_TXD_POPTS_TXSM || tso) { 1188 ckinfo[1].ck_start = 1189 cd->upper_setup.tcp_fields.tucss; 1190 ckinfo[1].ck_off = 1191 cd->upper_setup.tcp_fields.tucso; 1192 ckinfo[1].ck_len = 1193 cd->upper_setup.tcp_fields.tucse; 1194 } 1195 } 1196 break; 1197 } 1198 } 1199 1200 if (iovcnt > I82545_MAX_TXSEGS) { 1201 WPRINTF("tx too many descriptors (%d > %d) -- dropped\r\n", 1202 iovcnt, I82545_MAX_TXSEGS); 1203 goto done; 1204 } 1205 1206 hdrlen = vlen = 0; 1207 /* Estimate writable space for VLAN header insertion. */ 1208 if ((sc->esc_CTRL & E1000_CTRL_VME) && 1209 (dsc->td.lower.data & E1000_TXD_CMD_VLE)) { 1210 hdrlen = ETHER_ADDR_LEN*2; 1211 vlen = ETHER_VLAN_ENCAP_LEN; 1212 } 1213 if (!tso) { 1214 /* Estimate required writable space for checksums. */ 1215 if (ckinfo[0].ck_valid) 1216 hdrlen = MAX(hdrlen, ckinfo[0].ck_off + 2); 1217 if (ckinfo[1].ck_valid) 1218 hdrlen = MAX(hdrlen, ckinfo[1].ck_off + 2); 1219 /* Round up writable space to the first vector. */ 1220 if (hdrlen != 0 && iov[0].iov_len > hdrlen && 1221 iov[0].iov_len < hdrlen + 100) 1222 hdrlen = iov[0].iov_len; 1223 } else { 1224 /* In case of TSO header length provided by software. */ 1225 hdrlen = sc->esc_txctx.tcp_seg_setup.fields.hdr_len; 1226 } 1227 1228 /* Allocate, fill and prepend writable header vector. */ 1229 if (hdrlen != 0) { 1230 hdr = __builtin_alloca(hdrlen + vlen); 1231 hdr += vlen; 1232 for (left = hdrlen, hdrp = hdr; left > 0; 1233 left -= now, hdrp += now) { 1234 now = MIN(left, iov->iov_len); 1235 memcpy(hdrp, iov->iov_base, now); 1236 iov->iov_base += now; 1237 iov->iov_len -= now; 1238 if (iov->iov_len == 0) { 1239 iov++; 1240 iovcnt--; 1241 } 1242 } 1243 iov--; 1244 iovcnt++; 1245 iov->iov_base = hdr; 1246 iov->iov_len = hdrlen; 1247 } 1248 1249 /* Insert VLAN tag. */ 1250 if (vlen != 0) { 1251 hdr -= ETHER_VLAN_ENCAP_LEN; 1252 memmove(hdr, hdr + ETHER_VLAN_ENCAP_LEN, ETHER_ADDR_LEN*2); 1253 hdrlen += ETHER_VLAN_ENCAP_LEN; 1254 hdr[ETHER_ADDR_LEN*2 + 0] = sc->esc_VET >> 8; 1255 hdr[ETHER_ADDR_LEN*2 + 1] = sc->esc_VET & 0xff; 1256 hdr[ETHER_ADDR_LEN*2 + 2] = dsc->td.upper.fields.special >> 8; 1257 hdr[ETHER_ADDR_LEN*2 + 3] = dsc->td.upper.fields.special & 0xff; 1258 iov->iov_base = hdr; 1259 iov->iov_len += ETHER_VLAN_ENCAP_LEN; 1260 /* Correct checksum offsets after VLAN tag insertion. */ 1261 ckinfo[0].ck_start += ETHER_VLAN_ENCAP_LEN; 1262 ckinfo[0].ck_off += ETHER_VLAN_ENCAP_LEN; 1263 if (ckinfo[0].ck_len != 0) 1264 ckinfo[0].ck_len += ETHER_VLAN_ENCAP_LEN; 1265 ckinfo[1].ck_start += ETHER_VLAN_ENCAP_LEN; 1266 ckinfo[1].ck_off += ETHER_VLAN_ENCAP_LEN; 1267 if (ckinfo[1].ck_len != 0) 1268 ckinfo[1].ck_len += ETHER_VLAN_ENCAP_LEN; 1269 } 1270 1271 /* Simple non-TSO case. */ 1272 if (!tso) { 1273 /* Calculate checksums and transmit. */ 1274 if (ckinfo[0].ck_valid) 1275 e82545_transmit_checksum(iov, iovcnt, &ckinfo[0]); 1276 if (ckinfo[1].ck_valid) 1277 e82545_transmit_checksum(iov, iovcnt, &ckinfo[1]); 1278 e82545_transmit_backend(sc, iov, iovcnt); 1279 goto done; 1280 } 1281 1282 /* Doing TSO. */ 1283 tcp = (sc->esc_txctx.cmd_and_length & E1000_TXD_CMD_TCP) != 0; 1284 mss = sc->esc_txctx.tcp_seg_setup.fields.mss; 1285 paylen = (sc->esc_txctx.cmd_and_length & 0x000fffff); 1286 DPRINTF("tx %s segmentation offload %d+%d/%d bytes %d iovs\r\n", 1287 tcp ? "TCP" : "UDP", hdrlen, paylen, mss, iovcnt); 1288 ipid = ntohs(*(uint16_t *)&hdr[ckinfo[0].ck_start + 4]); 1289 tcpseq = ntohl(*(uint32_t *)&hdr[ckinfo[1].ck_start + 4]); 1290 ipcs = *(uint16_t *)&hdr[ckinfo[0].ck_off]; 1291 tcpcs = 0; 1292 if (ckinfo[1].ck_valid) /* Save partial pseudo-header checksum. */ 1293 tcpcs = *(uint16_t *)&hdr[ckinfo[1].ck_off]; 1294 pv = 1; 1295 pvoff = 0; 1296 for (seg = 0, left = paylen; left > 0; seg++, left -= now) { 1297 now = MIN(left, mss); 1298 1299 /* Construct IOVs for the segment. */ 1300 /* Include whole original header. */ 1301 tiov[0].iov_base = hdr; 1302 tiov[0].iov_len = hdrlen; 1303 tiovcnt = 1; 1304 /* Include respective part of payload IOV. */ 1305 for (nleft = now; pv < iovcnt && nleft > 0; nleft -= nnow) { 1306 nnow = MIN(nleft, iov[pv].iov_len - pvoff); 1307 tiov[tiovcnt].iov_base = iov[pv].iov_base + pvoff; 1308 tiov[tiovcnt++].iov_len = nnow; 1309 if (pvoff + nnow == iov[pv].iov_len) { 1310 pv++; 1311 pvoff = 0; 1312 } else 1313 pvoff += nnow; 1314 } 1315 DPRINTF("tx segment %d %d+%d bytes %d iovs\r\n", 1316 seg, hdrlen, now, tiovcnt); 1317 1318 /* Update IP header. */ 1319 if (sc->esc_txctx.cmd_and_length & E1000_TXD_CMD_IP) { 1320 /* IPv4 -- set length and ID */ 1321 *(uint16_t *)&hdr[ckinfo[0].ck_start + 2] = 1322 htons(hdrlen - ckinfo[0].ck_start + now); 1323 *(uint16_t *)&hdr[ckinfo[0].ck_start + 4] = 1324 htons(ipid + seg); 1325 } else { 1326 /* IPv6 -- set length */ 1327 *(uint16_t *)&hdr[ckinfo[0].ck_start + 4] = 1328 htons(hdrlen - ckinfo[0].ck_start - 40 + 1329 now); 1330 } 1331 1332 /* Update pseudo-header checksum. */ 1333 tcpsum = tcpcs; 1334 tcpsum += htons(hdrlen - ckinfo[1].ck_start + now); 1335 1336 /* Update TCP/UDP headers. */ 1337 if (tcp) { 1338 /* Update sequence number and FIN/PUSH flags. */ 1339 *(uint32_t *)&hdr[ckinfo[1].ck_start + 4] = 1340 htonl(tcpseq + paylen - left); 1341 if (now < left) { 1342 hdr[ckinfo[1].ck_start + 13] &= 1343 ~(TH_FIN | TH_PUSH); 1344 } 1345 } else { 1346 /* Update payload length. */ 1347 *(uint32_t *)&hdr[ckinfo[1].ck_start + 4] = 1348 hdrlen - ckinfo[1].ck_start + now; 1349 } 1350 1351 /* Calculate checksums and transmit. */ 1352 if (ckinfo[0].ck_valid) { 1353 *(uint16_t *)&hdr[ckinfo[0].ck_off] = ipcs; 1354 e82545_transmit_checksum(tiov, tiovcnt, &ckinfo[0]); 1355 } 1356 if (ckinfo[1].ck_valid) { 1357 *(uint16_t *)&hdr[ckinfo[1].ck_off] = 1358 e82545_carry(tcpsum); 1359 e82545_transmit_checksum(tiov, tiovcnt, &ckinfo[1]); 1360 } 1361 e82545_transmit_backend(sc, tiov, tiovcnt); 1362 } 1363 1364 done: 1365 head = (head + 1) % dsize; 1366 e82545_transmit_done(sc, ohead, head, dsize, tdwb); 1367 1368 *rhead = head; 1369 return (desc + 1); 1370 } 1371 1372 static void 1373 e82545_tx_run(struct e82545_softc *sc) 1374 { 1375 uint32_t cause; 1376 uint16_t head, rhead, tail, size; 1377 int lim, tdwb, sent; 1378 1379 head = sc->esc_TDH; 1380 tail = sc->esc_TDT; 1381 size = sc->esc_TDLEN / 16; 1382 DPRINTF("tx_run: head %x, rhead %x, tail %x\r\n", 1383 sc->esc_TDH, sc->esc_TDHr, sc->esc_TDT); 1384 1385 pthread_mutex_unlock(&sc->esc_mtx); 1386 rhead = head; 1387 tdwb = 0; 1388 for (lim = size / 4; sc->esc_tx_enabled && lim > 0; lim -= sent) { 1389 sent = e82545_transmit(sc, head, tail, size, &rhead, &tdwb); 1390 if (sent == 0) 1391 break; 1392 head = rhead; 1393 } 1394 pthread_mutex_lock(&sc->esc_mtx); 1395 1396 sc->esc_TDH = head; 1397 sc->esc_TDHr = rhead; 1398 cause = 0; 1399 if (tdwb) 1400 cause |= E1000_ICR_TXDW; 1401 if (lim != size / 4 && sc->esc_TDH == sc->esc_TDT) 1402 cause |= E1000_ICR_TXQE; 1403 if (cause) 1404 e82545_icr_assert(sc, cause); 1405 1406 DPRINTF("tx_run done: head %x, rhead %x, tail %x\r\n", 1407 sc->esc_TDH, sc->esc_TDHr, sc->esc_TDT); 1408 } 1409 1410 static _Noreturn void * 1411 e82545_tx_thread(void *param) 1412 { 1413 struct e82545_softc *sc = param; 1414 1415 pthread_mutex_lock(&sc->esc_mtx); 1416 for (;;) { 1417 while (!sc->esc_tx_enabled || sc->esc_TDHr == sc->esc_TDT) { 1418 if (sc->esc_tx_enabled && sc->esc_TDHr != sc->esc_TDT) 1419 break; 1420 sc->esc_tx_active = 0; 1421 if (sc->esc_tx_enabled == 0) 1422 pthread_cond_signal(&sc->esc_tx_cond); 1423 pthread_cond_wait(&sc->esc_tx_cond, &sc->esc_mtx); 1424 } 1425 sc->esc_tx_active = 1; 1426 1427 /* Process some tx descriptors. Lock dropped inside. */ 1428 e82545_tx_run(sc); 1429 } 1430 } 1431 1432 static void 1433 e82545_tx_start(struct e82545_softc *sc) 1434 { 1435 1436 if (sc->esc_tx_active == 0) 1437 pthread_cond_signal(&sc->esc_tx_cond); 1438 } 1439 1440 static void 1441 e82545_tx_enable(struct e82545_softc *sc) 1442 { 1443 1444 sc->esc_tx_enabled = 1; 1445 } 1446 1447 static void 1448 e82545_tx_disable(struct e82545_softc *sc) 1449 { 1450 1451 sc->esc_tx_enabled = 0; 1452 while (sc->esc_tx_active) 1453 pthread_cond_wait(&sc->esc_tx_cond, &sc->esc_mtx); 1454 } 1455 1456 static void 1457 e82545_rx_enable(struct e82545_softc *sc) 1458 { 1459 1460 sc->esc_rx_enabled = 1; 1461 } 1462 1463 static void 1464 e82545_rx_disable(struct e82545_softc *sc) 1465 { 1466 1467 sc->esc_rx_enabled = 0; 1468 while (sc->esc_rx_active) 1469 pthread_cond_wait(&sc->esc_rx_cond, &sc->esc_mtx); 1470 } 1471 1472 static void 1473 e82545_write_ra(struct e82545_softc *sc, int reg, uint32_t wval) 1474 { 1475 struct eth_uni *eu; 1476 int idx; 1477 1478 idx = reg >> 1; 1479 assert(idx < 15); 1480 1481 eu = &sc->esc_uni[idx]; 1482 1483 if (reg & 0x1) { 1484 /* RAH */ 1485 eu->eu_valid = ((wval & E1000_RAH_AV) == E1000_RAH_AV); 1486 eu->eu_addrsel = (wval >> 16) & 0x3; 1487 eu->eu_eth.octet[5] = wval >> 8; 1488 eu->eu_eth.octet[4] = wval; 1489 } else { 1490 /* RAL */ 1491 eu->eu_eth.octet[3] = wval >> 24; 1492 eu->eu_eth.octet[2] = wval >> 16; 1493 eu->eu_eth.octet[1] = wval >> 8; 1494 eu->eu_eth.octet[0] = wval; 1495 } 1496 } 1497 1498 static uint32_t 1499 e82545_read_ra(struct e82545_softc *sc, int reg) 1500 { 1501 struct eth_uni *eu; 1502 uint32_t retval; 1503 int idx; 1504 1505 idx = reg >> 1; 1506 assert(idx < 15); 1507 1508 eu = &sc->esc_uni[idx]; 1509 1510 if (reg & 0x1) { 1511 /* RAH */ 1512 retval = (eu->eu_valid << 31) | 1513 (eu->eu_addrsel << 16) | 1514 (eu->eu_eth.octet[5] << 8) | 1515 eu->eu_eth.octet[4]; 1516 } else { 1517 /* RAL */ 1518 retval = (eu->eu_eth.octet[3] << 24) | 1519 (eu->eu_eth.octet[2] << 16) | 1520 (eu->eu_eth.octet[1] << 8) | 1521 eu->eu_eth.octet[0]; 1522 } 1523 1524 return (retval); 1525 } 1526 1527 static void 1528 e82545_write_register(struct e82545_softc *sc, uint32_t offset, uint32_t value) 1529 { 1530 int ridx; 1531 1532 if (offset & 0x3) { 1533 DPRINTF("Unaligned register write offset:0x%x value:0x%x\r\n", offset, value); 1534 return; 1535 } 1536 DPRINTF("Register write: 0x%x value: 0x%x\r\n", offset, value); 1537 1538 switch (offset) { 1539 case E1000_CTRL: 1540 case E1000_CTRL_DUP: 1541 e82545_devctl(sc, value); 1542 break; 1543 case E1000_FCAL: 1544 sc->esc_FCAL = value; 1545 break; 1546 case E1000_FCAH: 1547 sc->esc_FCAH = value & ~0xFFFF0000; 1548 break; 1549 case E1000_FCT: 1550 sc->esc_FCT = value & ~0xFFFF0000; 1551 break; 1552 case E1000_VET: 1553 sc->esc_VET = value & ~0xFFFF0000; 1554 break; 1555 case E1000_FCTTV: 1556 sc->esc_FCTTV = value & ~0xFFFF0000; 1557 break; 1558 case E1000_LEDCTL: 1559 sc->esc_LEDCTL = value & ~0x30303000; 1560 break; 1561 case E1000_PBA: 1562 sc->esc_PBA = value & 0x0000FF80; 1563 break; 1564 case E1000_ICR: 1565 case E1000_ITR: 1566 case E1000_ICS: 1567 case E1000_IMS: 1568 case E1000_IMC: 1569 e82545_intr_write(sc, offset, value); 1570 break; 1571 case E1000_RCTL: 1572 e82545_rx_ctl(sc, value); 1573 break; 1574 case E1000_FCRTL: 1575 sc->esc_FCRTL = value & ~0xFFFF0007; 1576 break; 1577 case E1000_FCRTH: 1578 sc->esc_FCRTH = value & ~0xFFFF0007; 1579 break; 1580 case E1000_RDBAL(0): 1581 sc->esc_RDBAL = value & ~0xF; 1582 if (sc->esc_rx_enabled) { 1583 /* Apparently legal: update cached address */ 1584 e82545_rx_update_rdba(sc); 1585 } 1586 break; 1587 case E1000_RDBAH(0): 1588 assert(!sc->esc_rx_enabled); 1589 sc->esc_RDBAH = value; 1590 break; 1591 case E1000_RDLEN(0): 1592 assert(!sc->esc_rx_enabled); 1593 sc->esc_RDLEN = value & ~0xFFF0007F; 1594 break; 1595 case E1000_RDH(0): 1596 /* XXX should only ever be zero ? Range check ? */ 1597 sc->esc_RDH = value; 1598 break; 1599 case E1000_RDT(0): 1600 /* XXX if this opens up the rx ring, do something ? */ 1601 sc->esc_RDT = value; 1602 break; 1603 case E1000_RDTR: 1604 /* ignore FPD bit 31 */ 1605 sc->esc_RDTR = value & ~0xFFFF0000; 1606 break; 1607 case E1000_RXDCTL(0): 1608 sc->esc_RXDCTL = value & ~0xFEC0C0C0; 1609 break; 1610 case E1000_RADV: 1611 sc->esc_RADV = value & ~0xFFFF0000; 1612 break; 1613 case E1000_RSRPD: 1614 sc->esc_RSRPD = value & ~0xFFFFF000; 1615 break; 1616 case E1000_RXCSUM: 1617 sc->esc_RXCSUM = value & ~0xFFFFF800; 1618 break; 1619 case E1000_TXCW: 1620 sc->esc_TXCW = value & ~0x3FFF0000; 1621 break; 1622 case E1000_TCTL: 1623 e82545_tx_ctl(sc, value); 1624 break; 1625 case E1000_TIPG: 1626 sc->esc_TIPG = value; 1627 break; 1628 case E1000_AIT: 1629 sc->esc_AIT = value; 1630 break; 1631 case E1000_TDBAL(0): 1632 sc->esc_TDBAL = value & ~0xF; 1633 if (sc->esc_tx_enabled) { 1634 /* Apparently legal */ 1635 e82545_tx_update_tdba(sc); 1636 } 1637 break; 1638 case E1000_TDBAH(0): 1639 //assert(!sc->esc_tx_enabled); 1640 sc->esc_TDBAH = value; 1641 break; 1642 case E1000_TDLEN(0): 1643 //assert(!sc->esc_tx_enabled); 1644 sc->esc_TDLEN = value & ~0xFFF0007F; 1645 break; 1646 case E1000_TDH(0): 1647 //assert(!sc->esc_tx_enabled); 1648 /* XXX should only ever be zero ? Range check ? */ 1649 sc->esc_TDHr = sc->esc_TDH = value; 1650 break; 1651 case E1000_TDT(0): 1652 /* XXX range check ? */ 1653 sc->esc_TDT = value; 1654 if (sc->esc_tx_enabled) 1655 e82545_tx_start(sc); 1656 break; 1657 case E1000_TIDV: 1658 sc->esc_TIDV = value & ~0xFFFF0000; 1659 break; 1660 case E1000_TXDCTL(0): 1661 //assert(!sc->esc_tx_enabled); 1662 sc->esc_TXDCTL = value & ~0xC0C0C0; 1663 break; 1664 case E1000_TADV: 1665 sc->esc_TADV = value & ~0xFFFF0000; 1666 break; 1667 case E1000_RAL(0) ... E1000_RAH(15): 1668 /* convert to u32 offset */ 1669 ridx = (offset - E1000_RAL(0)) >> 2; 1670 e82545_write_ra(sc, ridx, value); 1671 break; 1672 case E1000_MTA ... (E1000_MTA + (127*4)): 1673 sc->esc_fmcast[(offset - E1000_MTA) >> 2] = value; 1674 break; 1675 case E1000_VFTA ... (E1000_VFTA + (127*4)): 1676 sc->esc_fvlan[(offset - E1000_VFTA) >> 2] = value; 1677 break; 1678 case E1000_EECD: 1679 { 1680 //DPRINTF("EECD write 0x%x -> 0x%x\r\n", sc->eeprom_control, value); 1681 /* edge triggered low->high */ 1682 uint32_t eecd_strobe = ((sc->eeprom_control & E1000_EECD_SK) ? 1683 0 : (value & E1000_EECD_SK)); 1684 uint32_t eecd_mask = (E1000_EECD_SK|E1000_EECD_CS| 1685 E1000_EECD_DI|E1000_EECD_REQ); 1686 sc->eeprom_control &= ~eecd_mask; 1687 sc->eeprom_control |= (value & eecd_mask); 1688 /* grant/revoke immediately */ 1689 if (value & E1000_EECD_REQ) { 1690 sc->eeprom_control |= E1000_EECD_GNT; 1691 } else { 1692 sc->eeprom_control &= ~E1000_EECD_GNT; 1693 } 1694 if (eecd_strobe && (sc->eeprom_control & E1000_EECD_CS)) { 1695 e82545_eecd_strobe(sc); 1696 } 1697 return; 1698 } 1699 case E1000_MDIC: 1700 { 1701 uint8_t reg_addr = (uint8_t)((value & E1000_MDIC_REG_MASK) >> 1702 E1000_MDIC_REG_SHIFT); 1703 uint8_t phy_addr = (uint8_t)((value & E1000_MDIC_PHY_MASK) >> 1704 E1000_MDIC_PHY_SHIFT); 1705 sc->mdi_control = 1706 (value & ~(E1000_MDIC_ERROR|E1000_MDIC_DEST)); 1707 if ((value & E1000_MDIC_READY) != 0) { 1708 DPRINTF("Incorrect MDIC ready bit: 0x%x\r\n", value); 1709 return; 1710 } 1711 switch (value & E82545_MDIC_OP_MASK) { 1712 case E1000_MDIC_OP_READ: 1713 sc->mdi_control &= ~E82545_MDIC_DATA_MASK; 1714 sc->mdi_control |= e82545_read_mdi(sc, reg_addr, phy_addr); 1715 break; 1716 case E1000_MDIC_OP_WRITE: 1717 e82545_write_mdi(sc, reg_addr, phy_addr, 1718 value & E82545_MDIC_DATA_MASK); 1719 break; 1720 default: 1721 DPRINTF("Unknown MDIC op: 0x%x\r\n", value); 1722 return; 1723 } 1724 /* TODO: barrier? */ 1725 sc->mdi_control |= E1000_MDIC_READY; 1726 if (value & E82545_MDIC_IE) { 1727 // TODO: generate interrupt 1728 } 1729 return; 1730 } 1731 case E1000_MANC: 1732 case E1000_STATUS: 1733 return; 1734 default: 1735 DPRINTF("Unknown write register: 0x%x value:%x\r\n", offset, value); 1736 return; 1737 } 1738 } 1739 1740 static uint32_t 1741 e82545_read_register(struct e82545_softc *sc, uint32_t offset) 1742 { 1743 uint32_t retval; 1744 int ridx; 1745 1746 if (offset & 0x3) { 1747 DPRINTF("Unaligned register read offset:0x%x\r\n", offset); 1748 return 0; 1749 } 1750 1751 DPRINTF("Register read: 0x%x\r\n", offset); 1752 1753 switch (offset) { 1754 case E1000_CTRL: 1755 retval = sc->esc_CTRL; 1756 break; 1757 case E1000_STATUS: 1758 retval = E1000_STATUS_FD | E1000_STATUS_LU | 1759 E1000_STATUS_SPEED_1000; 1760 break; 1761 case E1000_FCAL: 1762 retval = sc->esc_FCAL; 1763 break; 1764 case E1000_FCAH: 1765 retval = sc->esc_FCAH; 1766 break; 1767 case E1000_FCT: 1768 retval = sc->esc_FCT; 1769 break; 1770 case E1000_VET: 1771 retval = sc->esc_VET; 1772 break; 1773 case E1000_FCTTV: 1774 retval = sc->esc_FCTTV; 1775 break; 1776 case E1000_LEDCTL: 1777 retval = sc->esc_LEDCTL; 1778 break; 1779 case E1000_PBA: 1780 retval = sc->esc_PBA; 1781 break; 1782 case E1000_ICR: 1783 case E1000_ITR: 1784 case E1000_ICS: 1785 case E1000_IMS: 1786 case E1000_IMC: 1787 retval = e82545_intr_read(sc, offset); 1788 break; 1789 case E1000_RCTL: 1790 retval = sc->esc_RCTL; 1791 break; 1792 case E1000_FCRTL: 1793 retval = sc->esc_FCRTL; 1794 break; 1795 case E1000_FCRTH: 1796 retval = sc->esc_FCRTH; 1797 break; 1798 case E1000_RDBAL(0): 1799 retval = sc->esc_RDBAL; 1800 break; 1801 case E1000_RDBAH(0): 1802 retval = sc->esc_RDBAH; 1803 break; 1804 case E1000_RDLEN(0): 1805 retval = sc->esc_RDLEN; 1806 break; 1807 case E1000_RDH(0): 1808 retval = sc->esc_RDH; 1809 break; 1810 case E1000_RDT(0): 1811 retval = sc->esc_RDT; 1812 break; 1813 case E1000_RDTR: 1814 retval = sc->esc_RDTR; 1815 break; 1816 case E1000_RXDCTL(0): 1817 retval = sc->esc_RXDCTL; 1818 break; 1819 case E1000_RADV: 1820 retval = sc->esc_RADV; 1821 break; 1822 case E1000_RSRPD: 1823 retval = sc->esc_RSRPD; 1824 break; 1825 case E1000_RXCSUM: 1826 retval = sc->esc_RXCSUM; 1827 break; 1828 case E1000_TXCW: 1829 retval = sc->esc_TXCW; 1830 break; 1831 case E1000_TCTL: 1832 retval = sc->esc_TCTL; 1833 break; 1834 case E1000_TIPG: 1835 retval = sc->esc_TIPG; 1836 break; 1837 case E1000_AIT: 1838 retval = sc->esc_AIT; 1839 break; 1840 case E1000_TDBAL(0): 1841 retval = sc->esc_TDBAL; 1842 break; 1843 case E1000_TDBAH(0): 1844 retval = sc->esc_TDBAH; 1845 break; 1846 case E1000_TDLEN(0): 1847 retval = sc->esc_TDLEN; 1848 break; 1849 case E1000_TDH(0): 1850 retval = sc->esc_TDH; 1851 break; 1852 case E1000_TDT(0): 1853 retval = sc->esc_TDT; 1854 break; 1855 case E1000_TIDV: 1856 retval = sc->esc_TIDV; 1857 break; 1858 case E1000_TXDCTL(0): 1859 retval = sc->esc_TXDCTL; 1860 break; 1861 case E1000_TADV: 1862 retval = sc->esc_TADV; 1863 break; 1864 case E1000_RAL(0) ... E1000_RAH(15): 1865 /* convert to u32 offset */ 1866 ridx = (offset - E1000_RAL(0)) >> 2; 1867 retval = e82545_read_ra(sc, ridx); 1868 break; 1869 case E1000_MTA ... (E1000_MTA + (127*4)): 1870 retval = sc->esc_fmcast[(offset - E1000_MTA) >> 2]; 1871 break; 1872 case E1000_VFTA ... (E1000_VFTA + (127*4)): 1873 retval = sc->esc_fvlan[(offset - E1000_VFTA) >> 2]; 1874 break; 1875 case E1000_EECD: 1876 //DPRINTF("EECD read %x\r\n", sc->eeprom_control); 1877 retval = sc->eeprom_control; 1878 break; 1879 case E1000_MDIC: 1880 retval = sc->mdi_control; 1881 break; 1882 case E1000_MANC: 1883 retval = 0; 1884 break; 1885 /* stats that we emulate. */ 1886 case E1000_MPC: 1887 retval = sc->missed_pkt_count; 1888 break; 1889 case E1000_PRC64: 1890 retval = sc->pkt_rx_by_size[0]; 1891 break; 1892 case E1000_PRC127: 1893 retval = sc->pkt_rx_by_size[1]; 1894 break; 1895 case E1000_PRC255: 1896 retval = sc->pkt_rx_by_size[2]; 1897 break; 1898 case E1000_PRC511: 1899 retval = sc->pkt_rx_by_size[3]; 1900 break; 1901 case E1000_PRC1023: 1902 retval = sc->pkt_rx_by_size[4]; 1903 break; 1904 case E1000_PRC1522: 1905 retval = sc->pkt_rx_by_size[5]; 1906 break; 1907 case E1000_GPRC: 1908 retval = sc->good_pkt_rx_count; 1909 break; 1910 case E1000_BPRC: 1911 retval = sc->bcast_pkt_rx_count; 1912 break; 1913 case E1000_MPRC: 1914 retval = sc->mcast_pkt_rx_count; 1915 break; 1916 case E1000_GPTC: 1917 case E1000_TPT: 1918 retval = sc->good_pkt_tx_count; 1919 break; 1920 case E1000_GORCL: 1921 retval = (uint32_t)sc->good_octets_rx; 1922 break; 1923 case E1000_GORCH: 1924 retval = (uint32_t)(sc->good_octets_rx >> 32); 1925 break; 1926 case E1000_TOTL: 1927 case E1000_GOTCL: 1928 retval = (uint32_t)sc->good_octets_tx; 1929 break; 1930 case E1000_TOTH: 1931 case E1000_GOTCH: 1932 retval = (uint32_t)(sc->good_octets_tx >> 32); 1933 break; 1934 case E1000_ROC: 1935 retval = sc->oversize_rx_count; 1936 break; 1937 case E1000_TORL: 1938 retval = (uint32_t)(sc->good_octets_rx + sc->missed_octets); 1939 break; 1940 case E1000_TORH: 1941 retval = (uint32_t)((sc->good_octets_rx + 1942 sc->missed_octets) >> 32); 1943 break; 1944 case E1000_TPR: 1945 retval = sc->good_pkt_rx_count + sc->missed_pkt_count + 1946 sc->oversize_rx_count; 1947 break; 1948 case E1000_PTC64: 1949 retval = sc->pkt_tx_by_size[0]; 1950 break; 1951 case E1000_PTC127: 1952 retval = sc->pkt_tx_by_size[1]; 1953 break; 1954 case E1000_PTC255: 1955 retval = sc->pkt_tx_by_size[2]; 1956 break; 1957 case E1000_PTC511: 1958 retval = sc->pkt_tx_by_size[3]; 1959 break; 1960 case E1000_PTC1023: 1961 retval = sc->pkt_tx_by_size[4]; 1962 break; 1963 case E1000_PTC1522: 1964 retval = sc->pkt_tx_by_size[5]; 1965 break; 1966 case E1000_MPTC: 1967 retval = sc->mcast_pkt_tx_count; 1968 break; 1969 case E1000_BPTC: 1970 retval = sc->bcast_pkt_tx_count; 1971 break; 1972 case E1000_TSCTC: 1973 retval = sc->tso_tx_count; 1974 break; 1975 /* stats that are always 0. */ 1976 case E1000_CRCERRS: 1977 case E1000_ALGNERRC: 1978 case E1000_SYMERRS: 1979 case E1000_RXERRC: 1980 case E1000_SCC: 1981 case E1000_ECOL: 1982 case E1000_MCC: 1983 case E1000_LATECOL: 1984 case E1000_COLC: 1985 case E1000_DC: 1986 case E1000_TNCRS: 1987 case E1000_SEC: 1988 case E1000_CEXTERR: 1989 case E1000_RLEC: 1990 case E1000_XONRXC: 1991 case E1000_XONTXC: 1992 case E1000_XOFFRXC: 1993 case E1000_XOFFTXC: 1994 case E1000_FCRUC: 1995 case E1000_RNBC: 1996 case E1000_RUC: 1997 case E1000_RFC: 1998 case E1000_RJC: 1999 case E1000_MGTPRC: 2000 case E1000_MGTPDC: 2001 case E1000_MGTPTC: 2002 case E1000_TSCTFC: 2003 retval = 0; 2004 break; 2005 default: 2006 DPRINTF("Unknown read register: 0x%x\r\n", offset); 2007 retval = 0; 2008 break; 2009 } 2010 2011 return (retval); 2012 } 2013 2014 static void 2015 e82545_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx, 2016 uint64_t offset, int size, uint64_t value) 2017 { 2018 struct e82545_softc *sc; 2019 2020 //DPRINTF("Write bar:%d offset:0x%lx value:0x%lx size:%d\r\n", baridx, offset, value, size); 2021 2022 sc = pi->pi_arg; 2023 2024 pthread_mutex_lock(&sc->esc_mtx); 2025 2026 switch (baridx) { 2027 case E82545_BAR_IO: 2028 switch (offset) { 2029 case E82545_IOADDR: 2030 if (size != 4) { 2031 DPRINTF("Wrong io addr write sz:%d value:0x%lx\r\n", size, value); 2032 } else 2033 sc->io_addr = (uint32_t)value; 2034 break; 2035 case E82545_IODATA: 2036 if (size != 4) { 2037 DPRINTF("Wrong io data write size:%d value:0x%lx\r\n", size, value); 2038 } else if (sc->io_addr > E82545_IO_REGISTER_MAX) { 2039 DPRINTF("Non-register io write addr:0x%x value:0x%lx\r\n", sc->io_addr, value); 2040 } else 2041 e82545_write_register(sc, sc->io_addr, 2042 (uint32_t)value); 2043 break; 2044 default: 2045 DPRINTF("Unknown io bar write offset:0x%lx value:0x%lx size:%d\r\n", offset, value, size); 2046 break; 2047 } 2048 break; 2049 case E82545_BAR_REGISTER: 2050 if (size != 4) { 2051 DPRINTF("Wrong register write size:%d offset:0x%lx value:0x%lx\r\n", size, offset, value); 2052 } else 2053 e82545_write_register(sc, (uint32_t)offset, 2054 (uint32_t)value); 2055 break; 2056 default: 2057 DPRINTF("Unknown write bar:%d off:0x%lx val:0x%lx size:%d\r\n", 2058 baridx, offset, value, size); 2059 } 2060 2061 pthread_mutex_unlock(&sc->esc_mtx); 2062 } 2063 2064 static uint64_t 2065 e82545_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx, 2066 uint64_t offset, int size) 2067 { 2068 struct e82545_softc *sc; 2069 uint64_t retval; 2070 2071 //DPRINTF("Read bar:%d offset:0x%lx size:%d\r\n", baridx, offset, size); 2072 sc = pi->pi_arg; 2073 retval = 0; 2074 2075 pthread_mutex_lock(&sc->esc_mtx); 2076 2077 switch (baridx) { 2078 case E82545_BAR_IO: 2079 switch (offset) { 2080 case E82545_IOADDR: 2081 if (size != 4) { 2082 DPRINTF("Wrong io addr read sz:%d\r\n", size); 2083 } else 2084 retval = sc->io_addr; 2085 break; 2086 case E82545_IODATA: 2087 if (size != 4) { 2088 DPRINTF("Wrong io data read sz:%d\r\n", size); 2089 } 2090 if (sc->io_addr > E82545_IO_REGISTER_MAX) { 2091 DPRINTF("Non-register io read addr:0x%x\r\n", 2092 sc->io_addr); 2093 } else 2094 retval = e82545_read_register(sc, sc->io_addr); 2095 break; 2096 default: 2097 DPRINTF("Unknown io bar read offset:0x%lx size:%d\r\n", 2098 offset, size); 2099 break; 2100 } 2101 break; 2102 case E82545_BAR_REGISTER: 2103 if (size != 4) { 2104 DPRINTF("Wrong register read size:%d offset:0x%lx\r\n", 2105 size, offset); 2106 } else 2107 retval = e82545_read_register(sc, (uint32_t)offset); 2108 break; 2109 default: 2110 DPRINTF("Unknown read bar:%d offset:0x%lx size:%d\r\n", 2111 baridx, offset, size); 2112 break; 2113 } 2114 2115 pthread_mutex_unlock(&sc->esc_mtx); 2116 2117 return (retval); 2118 } 2119 2120 static void 2121 e82545_reset(struct e82545_softc *sc, int drvr) 2122 { 2123 int i; 2124 2125 e82545_rx_disable(sc); 2126 e82545_tx_disable(sc); 2127 2128 /* clear outstanding interrupts */ 2129 if (sc->esc_irq_asserted) 2130 pci_lintr_deassert(sc->esc_pi); 2131 2132 /* misc */ 2133 if (!drvr) { 2134 sc->esc_FCAL = 0; 2135 sc->esc_FCAH = 0; 2136 sc->esc_FCT = 0; 2137 sc->esc_VET = 0; 2138 sc->esc_FCTTV = 0; 2139 } 2140 sc->esc_LEDCTL = 0x07061302; 2141 sc->esc_PBA = 0x00100030; 2142 2143 /* start nvm in opcode mode. */ 2144 sc->nvm_opaddr = 0; 2145 sc->nvm_mode = E82545_NVM_MODE_OPADDR; 2146 sc->nvm_bits = E82545_NVM_OPADDR_BITS; 2147 sc->eeprom_control = E1000_EECD_PRES | E82545_EECD_FWE_EN; 2148 e82545_init_eeprom(sc); 2149 2150 /* interrupt */ 2151 sc->esc_ICR = 0; 2152 sc->esc_ITR = 250; 2153 sc->esc_ICS = 0; 2154 sc->esc_IMS = 0; 2155 sc->esc_IMC = 0; 2156 2157 /* L2 filters */ 2158 if (!drvr) { 2159 memset(sc->esc_fvlan, 0, sizeof(sc->esc_fvlan)); 2160 memset(sc->esc_fmcast, 0, sizeof(sc->esc_fmcast)); 2161 memset(sc->esc_uni, 0, sizeof(sc->esc_uni)); 2162 2163 /* XXX not necessary on 82545 ?? */ 2164 sc->esc_uni[0].eu_valid = 1; 2165 memcpy(sc->esc_uni[0].eu_eth.octet, sc->esc_mac.octet, 2166 ETHER_ADDR_LEN); 2167 } else { 2168 /* Clear RAH valid bits */ 2169 for (i = 0; i < 16; i++) 2170 sc->esc_uni[i].eu_valid = 0; 2171 } 2172 2173 /* receive */ 2174 if (!drvr) { 2175 sc->esc_RDBAL = 0; 2176 sc->esc_RDBAH = 0; 2177 } 2178 sc->esc_RCTL = 0; 2179 sc->esc_FCRTL = 0; 2180 sc->esc_FCRTH = 0; 2181 sc->esc_RDLEN = 0; 2182 sc->esc_RDH = 0; 2183 sc->esc_RDT = 0; 2184 sc->esc_RDTR = 0; 2185 sc->esc_RXDCTL = (1 << 24) | (1 << 16); /* default GRAN/WTHRESH */ 2186 sc->esc_RADV = 0; 2187 sc->esc_RXCSUM = 0; 2188 2189 /* transmit */ 2190 if (!drvr) { 2191 sc->esc_TDBAL = 0; 2192 sc->esc_TDBAH = 0; 2193 sc->esc_TIPG = 0; 2194 sc->esc_AIT = 0; 2195 sc->esc_TIDV = 0; 2196 sc->esc_TADV = 0; 2197 } 2198 sc->esc_tdba = 0; 2199 sc->esc_txdesc = NULL; 2200 sc->esc_TXCW = 0; 2201 sc->esc_TCTL = 0; 2202 sc->esc_TDLEN = 0; 2203 sc->esc_TDT = 0; 2204 sc->esc_TDHr = sc->esc_TDH = 0; 2205 sc->esc_TXDCTL = 0; 2206 } 2207 2208 static void 2209 e82545_open_tap(struct e82545_softc *sc, char *opts) 2210 { 2211 char tbuf[80]; 2212 #ifndef WITHOUT_CAPSICUM 2213 cap_rights_t rights; 2214 #endif 2215 2216 if (opts == NULL) { 2217 sc->esc_tapfd = -1; 2218 return; 2219 } 2220 2221 strcpy(tbuf, "/dev/"); 2222 strlcat(tbuf, opts, sizeof(tbuf)); 2223 2224 sc->esc_tapfd = open(tbuf, O_RDWR); 2225 if (sc->esc_tapfd == -1) { 2226 DPRINTF("unable to open tap device %s\n", opts); 2227 exit(4); 2228 } 2229 2230 /* 2231 * Set non-blocking and register for read 2232 * notifications with the event loop 2233 */ 2234 int opt = 1; 2235 if (ioctl(sc->esc_tapfd, FIONBIO, &opt) < 0) { 2236 WPRINTF("tap device O_NONBLOCK failed: %d\n", errno); 2237 close(sc->esc_tapfd); 2238 sc->esc_tapfd = -1; 2239 } 2240 2241 #ifndef WITHOUT_CAPSICUM 2242 cap_rights_init(&rights, CAP_EVENT, CAP_READ, CAP_WRITE); 2243 if (cap_rights_limit(sc->esc_tapfd, &rights) == -1 && errno != ENOSYS) 2244 errx(EX_OSERR, "Unable to apply rights for sandbox"); 2245 #endif 2246 2247 sc->esc_mevp = mevent_add(sc->esc_tapfd, 2248 EVF_READ, 2249 e82545_tap_callback, 2250 sc); 2251 if (sc->esc_mevp == NULL) { 2252 DPRINTF("Could not register mevent %d\n", EVF_READ); 2253 close(sc->esc_tapfd); 2254 sc->esc_tapfd = -1; 2255 } 2256 } 2257 2258 static int 2259 e82545_parsemac(char *mac_str, uint8_t *mac_addr) 2260 { 2261 struct ether_addr *ea; 2262 char *tmpstr; 2263 char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 }; 2264 2265 tmpstr = strsep(&mac_str,"="); 2266 if ((mac_str != NULL) && (!strcmp(tmpstr,"mac"))) { 2267 ea = ether_aton(mac_str); 2268 if (ea == NULL || ETHER_IS_MULTICAST(ea->octet) || 2269 memcmp(ea->octet, zero_addr, ETHER_ADDR_LEN) == 0) { 2270 fprintf(stderr, "Invalid MAC %s\n", mac_str); 2271 return (1); 2272 } else 2273 memcpy(mac_addr, ea->octet, ETHER_ADDR_LEN); 2274 } 2275 return (0); 2276 } 2277 2278 static int 2279 e82545_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) 2280 { 2281 DPRINTF("Loading with options: %s\r\n", opts); 2282 2283 MD5_CTX mdctx; 2284 unsigned char digest[16]; 2285 char nstr[80]; 2286 struct e82545_softc *sc; 2287 char *devname; 2288 char *vtopts; 2289 int mac_provided; 2290 2291 /* Setup our softc */ 2292 sc = calloc(1, sizeof(*sc)); 2293 2294 pi->pi_arg = sc; 2295 sc->esc_pi = pi; 2296 sc->esc_ctx = ctx; 2297 2298 pthread_mutex_init(&sc->esc_mtx, NULL); 2299 pthread_cond_init(&sc->esc_rx_cond, NULL); 2300 pthread_cond_init(&sc->esc_tx_cond, NULL); 2301 pthread_create(&sc->esc_tx_tid, NULL, e82545_tx_thread, sc); 2302 snprintf(nstr, sizeof(nstr), "e82545-%d:%d tx", pi->pi_slot, 2303 pi->pi_func); 2304 pthread_set_name_np(sc->esc_tx_tid, nstr); 2305 2306 pci_set_cfgdata16(pi, PCIR_DEVICE, E82545_DEV_ID_82545EM_COPPER); 2307 pci_set_cfgdata16(pi, PCIR_VENDOR, E82545_VENDOR_ID_INTEL); 2308 pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_NETWORK); 2309 pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_NETWORK_ETHERNET); 2310 pci_set_cfgdata16(pi, PCIR_SUBDEV_0, E82545_SUBDEV_ID); 2311 pci_set_cfgdata16(pi, PCIR_SUBVEND_0, E82545_VENDOR_ID_INTEL); 2312 2313 pci_set_cfgdata8(pi, PCIR_HDRTYPE, PCIM_HDRTYPE_NORMAL); 2314 pci_set_cfgdata8(pi, PCIR_INTPIN, 0x1); 2315 2316 /* TODO: this card also supports msi, but the freebsd driver for it 2317 * does not, so I have not implemented it. */ 2318 pci_lintr_request(pi); 2319 2320 pci_emul_alloc_bar(pi, E82545_BAR_REGISTER, PCIBAR_MEM32, 2321 E82545_BAR_REGISTER_LEN); 2322 pci_emul_alloc_bar(pi, E82545_BAR_FLASH, PCIBAR_MEM32, 2323 E82545_BAR_FLASH_LEN); 2324 pci_emul_alloc_bar(pi, E82545_BAR_IO, PCIBAR_IO, 2325 E82545_BAR_IO_LEN); 2326 2327 /* 2328 * Attempt to open the tap device and read the MAC address 2329 * if specified. Copied from virtio-net, slightly modified. 2330 */ 2331 mac_provided = 0; 2332 sc->esc_tapfd = -1; 2333 if (opts != NULL) { 2334 int err; 2335 2336 devname = vtopts = strdup(opts); 2337 (void) strsep(&vtopts, ","); 2338 2339 if (vtopts != NULL) { 2340 err = e82545_parsemac(vtopts, sc->esc_mac.octet); 2341 if (err != 0) { 2342 free(devname); 2343 return (err); 2344 } 2345 mac_provided = 1; 2346 } 2347 2348 if (strncmp(devname, "tap", 3) == 0 || 2349 strncmp(devname, "vmnet", 5) == 0) 2350 e82545_open_tap(sc, devname); 2351 2352 free(devname); 2353 } 2354 2355 /* 2356 * The default MAC address is the standard NetApp OUI of 00-a0-98, 2357 * followed by an MD5 of the PCI slot/func number and dev name 2358 */ 2359 if (!mac_provided) { 2360 snprintf(nstr, sizeof(nstr), "%d-%d-%s", pi->pi_slot, 2361 pi->pi_func, vmname); 2362 2363 MD5Init(&mdctx); 2364 MD5Update(&mdctx, nstr, strlen(nstr)); 2365 MD5Final(digest, &mdctx); 2366 2367 sc->esc_mac.octet[0] = 0x00; 2368 sc->esc_mac.octet[1] = 0xa0; 2369 sc->esc_mac.octet[2] = 0x98; 2370 sc->esc_mac.octet[3] = digest[0]; 2371 sc->esc_mac.octet[4] = digest[1]; 2372 sc->esc_mac.octet[5] = digest[2]; 2373 } 2374 2375 /* H/w initiated reset */ 2376 e82545_reset(sc, 0); 2377 2378 return (0); 2379 } 2380 2381 struct pci_devemu pci_de_e82545 = { 2382 .pe_emu = "e1000", 2383 .pe_init = e82545_init, 2384 .pe_barwrite = e82545_write, 2385 .pe_barread = e82545_read 2386 }; 2387 PCI_EMUL_SET(pci_de_e82545); 2388 2389