180b4ecc8SPaolo Bonzini /* 280b4ecc8SPaolo Bonzini * Copyright (c) 2007, Intel Corporation. 380b4ecc8SPaolo Bonzini * 480b4ecc8SPaolo Bonzini * This work is licensed under the terms of the GNU GPL, version 2. See 580b4ecc8SPaolo Bonzini * the COPYING file in the top-level directory. 680b4ecc8SPaolo Bonzini * 780b4ecc8SPaolo Bonzini * Jiang Yunhong <yunhong.jiang@intel.com> 880b4ecc8SPaolo Bonzini * 980b4ecc8SPaolo Bonzini * This file implements direct PCI assignment to a HVM guest 1080b4ecc8SPaolo Bonzini */ 1180b4ecc8SPaolo Bonzini 1280b4ecc8SPaolo Bonzini #include <sys/mman.h> 1380b4ecc8SPaolo Bonzini 1480b4ecc8SPaolo Bonzini #include "hw/xen/xen_backend.h" 1547b43a1fSPaolo Bonzini #include "xen_pt.h" 1680b4ecc8SPaolo Bonzini #include "hw/i386/apic-msidef.h" 1780b4ecc8SPaolo Bonzini 1880b4ecc8SPaolo Bonzini 1980b4ecc8SPaolo Bonzini #define XEN_PT_AUTO_ASSIGN -1 2080b4ecc8SPaolo Bonzini 2180b4ecc8SPaolo Bonzini /* shift count for gflags */ 2280b4ecc8SPaolo Bonzini #define XEN_PT_GFLAGS_SHIFT_DEST_ID 0 2380b4ecc8SPaolo Bonzini #define XEN_PT_GFLAGS_SHIFT_RH 8 2480b4ecc8SPaolo Bonzini #define XEN_PT_GFLAGS_SHIFT_DM 9 2580b4ecc8SPaolo Bonzini #define XEN_PT_GFLAGSSHIFT_DELIV_MODE 12 2680b4ecc8SPaolo Bonzini #define XEN_PT_GFLAGSSHIFT_TRG_MODE 15 2780b4ecc8SPaolo Bonzini 2880b4ecc8SPaolo Bonzini 2980b4ecc8SPaolo Bonzini /* 3080b4ecc8SPaolo Bonzini * Helpers 3180b4ecc8SPaolo Bonzini */ 3280b4ecc8SPaolo Bonzini 3380b4ecc8SPaolo Bonzini static inline uint8_t msi_vector(uint32_t data) 3480b4ecc8SPaolo Bonzini { 3580b4ecc8SPaolo Bonzini return (data & MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT; 3680b4ecc8SPaolo Bonzini } 3780b4ecc8SPaolo Bonzini 3880b4ecc8SPaolo Bonzini static inline uint8_t msi_dest_id(uint32_t addr) 3980b4ecc8SPaolo Bonzini { 4080b4ecc8SPaolo Bonzini return (addr & MSI_ADDR_DEST_ID_MASK) >> MSI_ADDR_DEST_ID_SHIFT; 4180b4ecc8SPaolo Bonzini } 4280b4ecc8SPaolo Bonzini 4380b4ecc8SPaolo Bonzini static inline uint32_t msi_ext_dest_id(uint32_t addr_hi) 4480b4ecc8SPaolo Bonzini { 4580b4ecc8SPaolo Bonzini return addr_hi & 0xffffff00; 4680b4ecc8SPaolo Bonzini } 4780b4ecc8SPaolo Bonzini 4880b4ecc8SPaolo Bonzini static uint32_t msi_gflags(uint32_t data, uint64_t addr) 4980b4ecc8SPaolo Bonzini { 5080b4ecc8SPaolo Bonzini uint32_t result = 0; 5180b4ecc8SPaolo Bonzini int rh, dm, dest_id, deliv_mode, trig_mode; 5280b4ecc8SPaolo Bonzini 5380b4ecc8SPaolo Bonzini rh = (addr >> MSI_ADDR_REDIRECTION_SHIFT) & 0x1; 5480b4ecc8SPaolo Bonzini dm = (addr >> MSI_ADDR_DEST_MODE_SHIFT) & 0x1; 5580b4ecc8SPaolo Bonzini dest_id = msi_dest_id(addr); 5680b4ecc8SPaolo Bonzini deliv_mode = (data >> MSI_DATA_DELIVERY_MODE_SHIFT) & 0x7; 5780b4ecc8SPaolo Bonzini trig_mode = (data >> MSI_DATA_TRIGGER_SHIFT) & 0x1; 5880b4ecc8SPaolo Bonzini 5980b4ecc8SPaolo Bonzini result = dest_id | (rh << XEN_PT_GFLAGS_SHIFT_RH) 6080b4ecc8SPaolo Bonzini | (dm << XEN_PT_GFLAGS_SHIFT_DM) 6180b4ecc8SPaolo Bonzini | (deliv_mode << XEN_PT_GFLAGSSHIFT_DELIV_MODE) 6280b4ecc8SPaolo Bonzini | (trig_mode << XEN_PT_GFLAGSSHIFT_TRG_MODE); 6380b4ecc8SPaolo Bonzini 6480b4ecc8SPaolo Bonzini return result; 6580b4ecc8SPaolo Bonzini } 6680b4ecc8SPaolo Bonzini 6780b4ecc8SPaolo Bonzini static inline uint64_t msi_addr64(XenPTMSI *msi) 6880b4ecc8SPaolo Bonzini { 6980b4ecc8SPaolo Bonzini return (uint64_t)msi->addr_hi << 32 | msi->addr_lo; 7080b4ecc8SPaolo Bonzini } 7180b4ecc8SPaolo Bonzini 7280b4ecc8SPaolo Bonzini static int msi_msix_enable(XenPCIPassthroughState *s, 7380b4ecc8SPaolo Bonzini uint32_t address, 7480b4ecc8SPaolo Bonzini uint16_t flag, 7580b4ecc8SPaolo Bonzini bool enable) 7680b4ecc8SPaolo Bonzini { 7780b4ecc8SPaolo Bonzini uint16_t val = 0; 78*fe2da64cSKonrad Rzeszutek Wilk int rc; 7980b4ecc8SPaolo Bonzini 8080b4ecc8SPaolo Bonzini if (!address) { 8180b4ecc8SPaolo Bonzini return -1; 8280b4ecc8SPaolo Bonzini } 8380b4ecc8SPaolo Bonzini 84*fe2da64cSKonrad Rzeszutek Wilk rc = xen_host_pci_get_word(&s->real_device, address, &val); 85*fe2da64cSKonrad Rzeszutek Wilk if (rc) { 86*fe2da64cSKonrad Rzeszutek Wilk XEN_PT_ERR(&s->dev, "Failed to read MSI/MSI-X register (0x%x), rc:%d\n", 87*fe2da64cSKonrad Rzeszutek Wilk address, rc); 88*fe2da64cSKonrad Rzeszutek Wilk return rc; 89*fe2da64cSKonrad Rzeszutek Wilk } 9080b4ecc8SPaolo Bonzini if (enable) { 9180b4ecc8SPaolo Bonzini val |= flag; 9280b4ecc8SPaolo Bonzini } else { 9380b4ecc8SPaolo Bonzini val &= ~flag; 9480b4ecc8SPaolo Bonzini } 95*fe2da64cSKonrad Rzeszutek Wilk rc = xen_host_pci_set_word(&s->real_device, address, val); 96*fe2da64cSKonrad Rzeszutek Wilk if (rc) { 97*fe2da64cSKonrad Rzeszutek Wilk XEN_PT_ERR(&s->dev, "Failed to write MSI/MSI-X register (0x%x), rc:%d\n", 98*fe2da64cSKonrad Rzeszutek Wilk address, rc); 99*fe2da64cSKonrad Rzeszutek Wilk } 100*fe2da64cSKonrad Rzeszutek Wilk return rc; 10180b4ecc8SPaolo Bonzini } 10280b4ecc8SPaolo Bonzini 10380b4ecc8SPaolo Bonzini static int msi_msix_setup(XenPCIPassthroughState *s, 10480b4ecc8SPaolo Bonzini uint64_t addr, 10580b4ecc8SPaolo Bonzini uint32_t data, 10680b4ecc8SPaolo Bonzini int *ppirq, 10780b4ecc8SPaolo Bonzini bool is_msix, 10880b4ecc8SPaolo Bonzini int msix_entry, 10980b4ecc8SPaolo Bonzini bool is_not_mapped) 11080b4ecc8SPaolo Bonzini { 11180b4ecc8SPaolo Bonzini uint8_t gvec = msi_vector(data); 11280b4ecc8SPaolo Bonzini int rc = 0; 11380b4ecc8SPaolo Bonzini 11480b4ecc8SPaolo Bonzini assert((!is_msix && msix_entry == 0) || is_msix); 11580b4ecc8SPaolo Bonzini 11680b4ecc8SPaolo Bonzini if (gvec == 0) { 11780b4ecc8SPaolo Bonzini /* if gvec is 0, the guest is asking for a particular pirq that 11880b4ecc8SPaolo Bonzini * is passed as dest_id */ 11980b4ecc8SPaolo Bonzini *ppirq = msi_ext_dest_id(addr >> 32) | msi_dest_id(addr); 12080b4ecc8SPaolo Bonzini if (!*ppirq) { 12180b4ecc8SPaolo Bonzini /* this probably identifies an misconfiguration of the guest, 12280b4ecc8SPaolo Bonzini * try the emulated path */ 12380b4ecc8SPaolo Bonzini *ppirq = XEN_PT_UNASSIGNED_PIRQ; 12480b4ecc8SPaolo Bonzini } else { 12580b4ecc8SPaolo Bonzini XEN_PT_LOG(&s->dev, "requested pirq %d for MSI%s" 12680b4ecc8SPaolo Bonzini " (vec: %#x, entry: %#x)\n", 12780b4ecc8SPaolo Bonzini *ppirq, is_msix ? "-X" : "", gvec, msix_entry); 12880b4ecc8SPaolo Bonzini } 12980b4ecc8SPaolo Bonzini } 13080b4ecc8SPaolo Bonzini 13180b4ecc8SPaolo Bonzini if (is_not_mapped) { 13280b4ecc8SPaolo Bonzini uint64_t table_base = 0; 13380b4ecc8SPaolo Bonzini 13480b4ecc8SPaolo Bonzini if (is_msix) { 13580b4ecc8SPaolo Bonzini table_base = s->msix->table_base; 13680b4ecc8SPaolo Bonzini } 13780b4ecc8SPaolo Bonzini 13880b4ecc8SPaolo Bonzini rc = xc_physdev_map_pirq_msi(xen_xc, xen_domid, XEN_PT_AUTO_ASSIGN, 13980b4ecc8SPaolo Bonzini ppirq, PCI_DEVFN(s->real_device.dev, 14080b4ecc8SPaolo Bonzini s->real_device.func), 14180b4ecc8SPaolo Bonzini s->real_device.bus, 14280b4ecc8SPaolo Bonzini msix_entry, table_base); 14380b4ecc8SPaolo Bonzini if (rc) { 14480b4ecc8SPaolo Bonzini XEN_PT_ERR(&s->dev, 1453782f60dSJan Beulich "Mapping of MSI%s (err: %i, vec: %#x, entry %#x)\n", 1463782f60dSJan Beulich is_msix ? "-X" : "", errno, gvec, msix_entry); 14780b4ecc8SPaolo Bonzini return rc; 14880b4ecc8SPaolo Bonzini } 14980b4ecc8SPaolo Bonzini } 15080b4ecc8SPaolo Bonzini 15180b4ecc8SPaolo Bonzini return 0; 15280b4ecc8SPaolo Bonzini } 15380b4ecc8SPaolo Bonzini static int msi_msix_update(XenPCIPassthroughState *s, 15480b4ecc8SPaolo Bonzini uint64_t addr, 15580b4ecc8SPaolo Bonzini uint32_t data, 15680b4ecc8SPaolo Bonzini int pirq, 15780b4ecc8SPaolo Bonzini bool is_msix, 15880b4ecc8SPaolo Bonzini int msix_entry, 15980b4ecc8SPaolo Bonzini int *old_pirq) 16080b4ecc8SPaolo Bonzini { 16180b4ecc8SPaolo Bonzini PCIDevice *d = &s->dev; 16280b4ecc8SPaolo Bonzini uint8_t gvec = msi_vector(data); 16380b4ecc8SPaolo Bonzini uint32_t gflags = msi_gflags(data, addr); 16480b4ecc8SPaolo Bonzini int rc = 0; 16580b4ecc8SPaolo Bonzini uint64_t table_addr = 0; 16680b4ecc8SPaolo Bonzini 16780b4ecc8SPaolo Bonzini XEN_PT_LOG(d, "Updating MSI%s with pirq %d gvec %#x gflags %#x" 16880b4ecc8SPaolo Bonzini " (entry: %#x)\n", 16980b4ecc8SPaolo Bonzini is_msix ? "-X" : "", pirq, gvec, gflags, msix_entry); 17080b4ecc8SPaolo Bonzini 17180b4ecc8SPaolo Bonzini if (is_msix) { 17280b4ecc8SPaolo Bonzini table_addr = s->msix->mmio_base_addr; 17380b4ecc8SPaolo Bonzini } 17480b4ecc8SPaolo Bonzini 17580b4ecc8SPaolo Bonzini rc = xc_domain_update_msi_irq(xen_xc, xen_domid, gvec, 17680b4ecc8SPaolo Bonzini pirq, gflags, table_addr); 17780b4ecc8SPaolo Bonzini 17880b4ecc8SPaolo Bonzini if (rc) { 1793782f60dSJan Beulich XEN_PT_ERR(d, "Updating of MSI%s failed. (err: %d)\n", 1803782f60dSJan Beulich is_msix ? "-X" : "", errno); 18180b4ecc8SPaolo Bonzini 18280b4ecc8SPaolo Bonzini if (xc_physdev_unmap_pirq(xen_xc, xen_domid, *old_pirq)) { 1833782f60dSJan Beulich XEN_PT_ERR(d, "Unmapping of MSI%s pirq %d failed. (err: %d)\n", 1843782f60dSJan Beulich is_msix ? "-X" : "", *old_pirq, errno); 18580b4ecc8SPaolo Bonzini } 18680b4ecc8SPaolo Bonzini *old_pirq = XEN_PT_UNASSIGNED_PIRQ; 18780b4ecc8SPaolo Bonzini } 18880b4ecc8SPaolo Bonzini return rc; 18980b4ecc8SPaolo Bonzini } 19080b4ecc8SPaolo Bonzini 19180b4ecc8SPaolo Bonzini static int msi_msix_disable(XenPCIPassthroughState *s, 19280b4ecc8SPaolo Bonzini uint64_t addr, 19380b4ecc8SPaolo Bonzini uint32_t data, 19480b4ecc8SPaolo Bonzini int pirq, 19580b4ecc8SPaolo Bonzini bool is_msix, 19680b4ecc8SPaolo Bonzini bool is_binded) 19780b4ecc8SPaolo Bonzini { 19880b4ecc8SPaolo Bonzini PCIDevice *d = &s->dev; 19980b4ecc8SPaolo Bonzini uint8_t gvec = msi_vector(data); 20080b4ecc8SPaolo Bonzini uint32_t gflags = msi_gflags(data, addr); 20180b4ecc8SPaolo Bonzini int rc = 0; 20280b4ecc8SPaolo Bonzini 20380b4ecc8SPaolo Bonzini if (pirq == XEN_PT_UNASSIGNED_PIRQ) { 20480b4ecc8SPaolo Bonzini return 0; 20580b4ecc8SPaolo Bonzini } 20680b4ecc8SPaolo Bonzini 20780b4ecc8SPaolo Bonzini if (is_binded) { 20880b4ecc8SPaolo Bonzini XEN_PT_LOG(d, "Unbind MSI%s with pirq %d, gvec %#x\n", 20980b4ecc8SPaolo Bonzini is_msix ? "-X" : "", pirq, gvec); 21080b4ecc8SPaolo Bonzini rc = xc_domain_unbind_msi_irq(xen_xc, xen_domid, gvec, pirq, gflags); 21180b4ecc8SPaolo Bonzini if (rc) { 2123782f60dSJan Beulich XEN_PT_ERR(d, "Unbinding of MSI%s failed. (err: %d, pirq: %d, gvec: %#x)\n", 2133782f60dSJan Beulich is_msix ? "-X" : "", errno, pirq, gvec); 21480b4ecc8SPaolo Bonzini return rc; 21580b4ecc8SPaolo Bonzini } 21680b4ecc8SPaolo Bonzini } 21780b4ecc8SPaolo Bonzini 21880b4ecc8SPaolo Bonzini XEN_PT_LOG(d, "Unmap MSI%s pirq %d\n", is_msix ? "-X" : "", pirq); 21980b4ecc8SPaolo Bonzini rc = xc_physdev_unmap_pirq(xen_xc, xen_domid, pirq); 22080b4ecc8SPaolo Bonzini if (rc) { 2213782f60dSJan Beulich XEN_PT_ERR(d, "Unmapping of MSI%s pirq %d failed. (err: %i)\n", 2223782f60dSJan Beulich is_msix ? "-X" : "", pirq, errno); 22380b4ecc8SPaolo Bonzini return rc; 22480b4ecc8SPaolo Bonzini } 22580b4ecc8SPaolo Bonzini 22680b4ecc8SPaolo Bonzini return 0; 22780b4ecc8SPaolo Bonzini } 22880b4ecc8SPaolo Bonzini 22980b4ecc8SPaolo Bonzini /* 23080b4ecc8SPaolo Bonzini * MSI virtualization functions 23180b4ecc8SPaolo Bonzini */ 23280b4ecc8SPaolo Bonzini 233cf8124f0SKonrad Rzeszutek Wilk static int xen_pt_msi_set_enable(XenPCIPassthroughState *s, bool enable) 23480b4ecc8SPaolo Bonzini { 23580b4ecc8SPaolo Bonzini XEN_PT_LOG(&s->dev, "%s MSI.\n", enable ? "enabling" : "disabling"); 23680b4ecc8SPaolo Bonzini 23780b4ecc8SPaolo Bonzini if (!s->msi) { 23880b4ecc8SPaolo Bonzini return -1; 23980b4ecc8SPaolo Bonzini } 24080b4ecc8SPaolo Bonzini 24180b4ecc8SPaolo Bonzini return msi_msix_enable(s, s->msi->ctrl_offset, PCI_MSI_FLAGS_ENABLE, 24280b4ecc8SPaolo Bonzini enable); 24380b4ecc8SPaolo Bonzini } 24480b4ecc8SPaolo Bonzini 24580b4ecc8SPaolo Bonzini /* setup physical msi, but don't enable it */ 24680b4ecc8SPaolo Bonzini int xen_pt_msi_setup(XenPCIPassthroughState *s) 24780b4ecc8SPaolo Bonzini { 24880b4ecc8SPaolo Bonzini int pirq = XEN_PT_UNASSIGNED_PIRQ; 24980b4ecc8SPaolo Bonzini int rc = 0; 25080b4ecc8SPaolo Bonzini XenPTMSI *msi = s->msi; 25180b4ecc8SPaolo Bonzini 25280b4ecc8SPaolo Bonzini if (msi->initialized) { 25380b4ecc8SPaolo Bonzini XEN_PT_ERR(&s->dev, 25480b4ecc8SPaolo Bonzini "Setup physical MSI when it has been properly initialized.\n"); 25580b4ecc8SPaolo Bonzini return -1; 25680b4ecc8SPaolo Bonzini } 25780b4ecc8SPaolo Bonzini 25880b4ecc8SPaolo Bonzini rc = msi_msix_setup(s, msi_addr64(msi), msi->data, &pirq, false, 0, true); 25980b4ecc8SPaolo Bonzini if (rc) { 26080b4ecc8SPaolo Bonzini return rc; 26180b4ecc8SPaolo Bonzini } 26280b4ecc8SPaolo Bonzini 26380b4ecc8SPaolo Bonzini if (pirq < 0) { 26480b4ecc8SPaolo Bonzini XEN_PT_ERR(&s->dev, "Invalid pirq number: %d.\n", pirq); 26580b4ecc8SPaolo Bonzini return -1; 26680b4ecc8SPaolo Bonzini } 26780b4ecc8SPaolo Bonzini 26880b4ecc8SPaolo Bonzini msi->pirq = pirq; 26980b4ecc8SPaolo Bonzini XEN_PT_LOG(&s->dev, "MSI mapped with pirq %d.\n", pirq); 27080b4ecc8SPaolo Bonzini 27180b4ecc8SPaolo Bonzini return 0; 27280b4ecc8SPaolo Bonzini } 27380b4ecc8SPaolo Bonzini 27480b4ecc8SPaolo Bonzini int xen_pt_msi_update(XenPCIPassthroughState *s) 27580b4ecc8SPaolo Bonzini { 27680b4ecc8SPaolo Bonzini XenPTMSI *msi = s->msi; 27780b4ecc8SPaolo Bonzini return msi_msix_update(s, msi_addr64(msi), msi->data, msi->pirq, 27880b4ecc8SPaolo Bonzini false, 0, &msi->pirq); 27980b4ecc8SPaolo Bonzini } 28080b4ecc8SPaolo Bonzini 28180b4ecc8SPaolo Bonzini void xen_pt_msi_disable(XenPCIPassthroughState *s) 28280b4ecc8SPaolo Bonzini { 28380b4ecc8SPaolo Bonzini XenPTMSI *msi = s->msi; 28480b4ecc8SPaolo Bonzini 28580b4ecc8SPaolo Bonzini if (!msi) { 28680b4ecc8SPaolo Bonzini return; 28780b4ecc8SPaolo Bonzini } 28880b4ecc8SPaolo Bonzini 289*fe2da64cSKonrad Rzeszutek Wilk (void)xen_pt_msi_set_enable(s, false); 29080b4ecc8SPaolo Bonzini 29180b4ecc8SPaolo Bonzini msi_msix_disable(s, msi_addr64(msi), msi->data, msi->pirq, false, 29280b4ecc8SPaolo Bonzini msi->initialized); 29380b4ecc8SPaolo Bonzini 29480b4ecc8SPaolo Bonzini /* clear msi info */ 295c976437cSZhenzhong Duan msi->flags &= ~PCI_MSI_FLAGS_ENABLE; 296c976437cSZhenzhong Duan msi->initialized = false; 29780b4ecc8SPaolo Bonzini msi->mapped = false; 29880b4ecc8SPaolo Bonzini msi->pirq = XEN_PT_UNASSIGNED_PIRQ; 29980b4ecc8SPaolo Bonzini } 30080b4ecc8SPaolo Bonzini 30180b4ecc8SPaolo Bonzini /* 30280b4ecc8SPaolo Bonzini * MSI-X virtualization functions 30380b4ecc8SPaolo Bonzini */ 30480b4ecc8SPaolo Bonzini 30580b4ecc8SPaolo Bonzini static int msix_set_enable(XenPCIPassthroughState *s, bool enabled) 30680b4ecc8SPaolo Bonzini { 30780b4ecc8SPaolo Bonzini XEN_PT_LOG(&s->dev, "%s MSI-X.\n", enabled ? "enabling" : "disabling"); 30880b4ecc8SPaolo Bonzini 30980b4ecc8SPaolo Bonzini if (!s->msix) { 31080b4ecc8SPaolo Bonzini return -1; 31180b4ecc8SPaolo Bonzini } 31280b4ecc8SPaolo Bonzini 31380b4ecc8SPaolo Bonzini return msi_msix_enable(s, s->msix->ctrl_offset, PCI_MSIX_FLAGS_ENABLE, 31480b4ecc8SPaolo Bonzini enabled); 31580b4ecc8SPaolo Bonzini } 31680b4ecc8SPaolo Bonzini 31780b4ecc8SPaolo Bonzini static int xen_pt_msix_update_one(XenPCIPassthroughState *s, int entry_nr) 31880b4ecc8SPaolo Bonzini { 31980b4ecc8SPaolo Bonzini XenPTMSIXEntry *entry = NULL; 32080b4ecc8SPaolo Bonzini int pirq; 32180b4ecc8SPaolo Bonzini int rc; 32280b4ecc8SPaolo Bonzini 32380b4ecc8SPaolo Bonzini if (entry_nr < 0 || entry_nr >= s->msix->total_entries) { 32480b4ecc8SPaolo Bonzini return -EINVAL; 32580b4ecc8SPaolo Bonzini } 32680b4ecc8SPaolo Bonzini 32780b4ecc8SPaolo Bonzini entry = &s->msix->msix_entry[entry_nr]; 32880b4ecc8SPaolo Bonzini 32980b4ecc8SPaolo Bonzini if (!entry->updated) { 33080b4ecc8SPaolo Bonzini return 0; 33180b4ecc8SPaolo Bonzini } 33280b4ecc8SPaolo Bonzini 33380b4ecc8SPaolo Bonzini pirq = entry->pirq; 33480b4ecc8SPaolo Bonzini 33580b4ecc8SPaolo Bonzini rc = msi_msix_setup(s, entry->addr, entry->data, &pirq, true, entry_nr, 33680b4ecc8SPaolo Bonzini entry->pirq == XEN_PT_UNASSIGNED_PIRQ); 33780b4ecc8SPaolo Bonzini if (rc) { 33880b4ecc8SPaolo Bonzini return rc; 33980b4ecc8SPaolo Bonzini } 34080b4ecc8SPaolo Bonzini if (entry->pirq == XEN_PT_UNASSIGNED_PIRQ) { 34180b4ecc8SPaolo Bonzini entry->pirq = pirq; 34280b4ecc8SPaolo Bonzini } 34380b4ecc8SPaolo Bonzini 34480b4ecc8SPaolo Bonzini rc = msi_msix_update(s, entry->addr, entry->data, pirq, true, 34580b4ecc8SPaolo Bonzini entry_nr, &entry->pirq); 34680b4ecc8SPaolo Bonzini 34780b4ecc8SPaolo Bonzini if (!rc) { 34880b4ecc8SPaolo Bonzini entry->updated = false; 34980b4ecc8SPaolo Bonzini } 35080b4ecc8SPaolo Bonzini 35180b4ecc8SPaolo Bonzini return rc; 35280b4ecc8SPaolo Bonzini } 35380b4ecc8SPaolo Bonzini 35480b4ecc8SPaolo Bonzini int xen_pt_msix_update(XenPCIPassthroughState *s) 35580b4ecc8SPaolo Bonzini { 35680b4ecc8SPaolo Bonzini XenPTMSIX *msix = s->msix; 35780b4ecc8SPaolo Bonzini int i; 35880b4ecc8SPaolo Bonzini 35980b4ecc8SPaolo Bonzini for (i = 0; i < msix->total_entries; i++) { 36080b4ecc8SPaolo Bonzini xen_pt_msix_update_one(s, i); 36180b4ecc8SPaolo Bonzini } 36280b4ecc8SPaolo Bonzini 36380b4ecc8SPaolo Bonzini return 0; 36480b4ecc8SPaolo Bonzini } 36580b4ecc8SPaolo Bonzini 36680b4ecc8SPaolo Bonzini void xen_pt_msix_disable(XenPCIPassthroughState *s) 36780b4ecc8SPaolo Bonzini { 36880b4ecc8SPaolo Bonzini int i = 0; 36980b4ecc8SPaolo Bonzini 37080b4ecc8SPaolo Bonzini msix_set_enable(s, false); 37180b4ecc8SPaolo Bonzini 37280b4ecc8SPaolo Bonzini for (i = 0; i < s->msix->total_entries; i++) { 37380b4ecc8SPaolo Bonzini XenPTMSIXEntry *entry = &s->msix->msix_entry[i]; 37480b4ecc8SPaolo Bonzini 37580b4ecc8SPaolo Bonzini msi_msix_disable(s, entry->addr, entry->data, entry->pirq, true, true); 37680b4ecc8SPaolo Bonzini 37780b4ecc8SPaolo Bonzini /* clear MSI-X info */ 37880b4ecc8SPaolo Bonzini entry->pirq = XEN_PT_UNASSIGNED_PIRQ; 37980b4ecc8SPaolo Bonzini entry->updated = false; 38080b4ecc8SPaolo Bonzini } 38180b4ecc8SPaolo Bonzini } 38280b4ecc8SPaolo Bonzini 38380b4ecc8SPaolo Bonzini int xen_pt_msix_update_remap(XenPCIPassthroughState *s, int bar_index) 38480b4ecc8SPaolo Bonzini { 38580b4ecc8SPaolo Bonzini XenPTMSIXEntry *entry; 38680b4ecc8SPaolo Bonzini int i, ret; 38780b4ecc8SPaolo Bonzini 38880b4ecc8SPaolo Bonzini if (!(s->msix && s->msix->bar_index == bar_index)) { 38980b4ecc8SPaolo Bonzini return 0; 39080b4ecc8SPaolo Bonzini } 39180b4ecc8SPaolo Bonzini 39280b4ecc8SPaolo Bonzini for (i = 0; i < s->msix->total_entries; i++) { 39380b4ecc8SPaolo Bonzini entry = &s->msix->msix_entry[i]; 39480b4ecc8SPaolo Bonzini if (entry->pirq != XEN_PT_UNASSIGNED_PIRQ) { 39580b4ecc8SPaolo Bonzini ret = xc_domain_unbind_pt_irq(xen_xc, xen_domid, entry->pirq, 39680b4ecc8SPaolo Bonzini PT_IRQ_TYPE_MSI, 0, 0, 0, 0); 39780b4ecc8SPaolo Bonzini if (ret) { 3983782f60dSJan Beulich XEN_PT_ERR(&s->dev, "unbind MSI-X entry %d failed (err: %d)\n", 3993782f60dSJan Beulich entry->pirq, errno); 40080b4ecc8SPaolo Bonzini } 40180b4ecc8SPaolo Bonzini entry->updated = true; 40280b4ecc8SPaolo Bonzini } 40380b4ecc8SPaolo Bonzini } 40480b4ecc8SPaolo Bonzini return xen_pt_msix_update(s); 40580b4ecc8SPaolo Bonzini } 40680b4ecc8SPaolo Bonzini 40780b4ecc8SPaolo Bonzini static uint32_t get_entry_value(XenPTMSIXEntry *e, int offset) 40880b4ecc8SPaolo Bonzini { 40980b4ecc8SPaolo Bonzini switch (offset) { 41080b4ecc8SPaolo Bonzini case PCI_MSIX_ENTRY_LOWER_ADDR: 41180b4ecc8SPaolo Bonzini return e->addr & UINT32_MAX; 41280b4ecc8SPaolo Bonzini case PCI_MSIX_ENTRY_UPPER_ADDR: 41380b4ecc8SPaolo Bonzini return e->addr >> 32; 41480b4ecc8SPaolo Bonzini case PCI_MSIX_ENTRY_DATA: 41580b4ecc8SPaolo Bonzini return e->data; 41680b4ecc8SPaolo Bonzini case PCI_MSIX_ENTRY_VECTOR_CTRL: 41780b4ecc8SPaolo Bonzini return e->vector_ctrl; 41880b4ecc8SPaolo Bonzini default: 41980b4ecc8SPaolo Bonzini return 0; 42080b4ecc8SPaolo Bonzini } 42180b4ecc8SPaolo Bonzini } 42280b4ecc8SPaolo Bonzini 42380b4ecc8SPaolo Bonzini static void set_entry_value(XenPTMSIXEntry *e, int offset, uint32_t val) 42480b4ecc8SPaolo Bonzini { 42580b4ecc8SPaolo Bonzini switch (offset) { 42680b4ecc8SPaolo Bonzini case PCI_MSIX_ENTRY_LOWER_ADDR: 42780b4ecc8SPaolo Bonzini e->addr = (e->addr & ((uint64_t)UINT32_MAX << 32)) | val; 42880b4ecc8SPaolo Bonzini break; 42980b4ecc8SPaolo Bonzini case PCI_MSIX_ENTRY_UPPER_ADDR: 43080b4ecc8SPaolo Bonzini e->addr = (uint64_t)val << 32 | (e->addr & UINT32_MAX); 43180b4ecc8SPaolo Bonzini break; 43280b4ecc8SPaolo Bonzini case PCI_MSIX_ENTRY_DATA: 43380b4ecc8SPaolo Bonzini e->data = val; 43480b4ecc8SPaolo Bonzini break; 43580b4ecc8SPaolo Bonzini case PCI_MSIX_ENTRY_VECTOR_CTRL: 43680b4ecc8SPaolo Bonzini e->vector_ctrl = val; 43780b4ecc8SPaolo Bonzini break; 43880b4ecc8SPaolo Bonzini } 43980b4ecc8SPaolo Bonzini } 44080b4ecc8SPaolo Bonzini 44180b4ecc8SPaolo Bonzini static void pci_msix_write(void *opaque, hwaddr addr, 44280b4ecc8SPaolo Bonzini uint64_t val, unsigned size) 44380b4ecc8SPaolo Bonzini { 44480b4ecc8SPaolo Bonzini XenPCIPassthroughState *s = opaque; 44580b4ecc8SPaolo Bonzini XenPTMSIX *msix = s->msix; 44680b4ecc8SPaolo Bonzini XenPTMSIXEntry *entry; 447b38ec5eeSJan Beulich unsigned int entry_nr, offset; 44880b4ecc8SPaolo Bonzini 44980b4ecc8SPaolo Bonzini entry_nr = addr / PCI_MSIX_ENTRY_SIZE; 450b38ec5eeSJan Beulich if (entry_nr >= msix->total_entries) { 45180b4ecc8SPaolo Bonzini return; 45280b4ecc8SPaolo Bonzini } 45380b4ecc8SPaolo Bonzini entry = &msix->msix_entry[entry_nr]; 45480b4ecc8SPaolo Bonzini offset = addr % PCI_MSIX_ENTRY_SIZE; 45580b4ecc8SPaolo Bonzini 45680b4ecc8SPaolo Bonzini if (offset != PCI_MSIX_ENTRY_VECTOR_CTRL) { 45780b4ecc8SPaolo Bonzini const volatile uint32_t *vec_ctrl; 45880b4ecc8SPaolo Bonzini 459c976437cSZhenzhong Duan if (get_entry_value(entry, offset) == val 460c976437cSZhenzhong Duan && entry->pirq != XEN_PT_UNASSIGNED_PIRQ) { 46180b4ecc8SPaolo Bonzini return; 46280b4ecc8SPaolo Bonzini } 46380b4ecc8SPaolo Bonzini 46480b4ecc8SPaolo Bonzini /* 46580b4ecc8SPaolo Bonzini * If Xen intercepts the mask bit access, entry->vec_ctrl may not be 46680b4ecc8SPaolo Bonzini * up-to-date. Read from hardware directly. 46780b4ecc8SPaolo Bonzini */ 46880b4ecc8SPaolo Bonzini vec_ctrl = s->msix->phys_iomem_base + entry_nr * PCI_MSIX_ENTRY_SIZE 46980b4ecc8SPaolo Bonzini + PCI_MSIX_ENTRY_VECTOR_CTRL; 47080b4ecc8SPaolo Bonzini 47180b4ecc8SPaolo Bonzini if (msix->enabled && !(*vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT)) { 472b38ec5eeSJan Beulich if (!entry->warned) { 473b38ec5eeSJan Beulich entry->warned = true; 47480b4ecc8SPaolo Bonzini XEN_PT_ERR(&s->dev, "Can't update msix entry %d since MSI-X is" 47580b4ecc8SPaolo Bonzini " already enabled.\n", entry_nr); 476b38ec5eeSJan Beulich } 47780b4ecc8SPaolo Bonzini return; 47880b4ecc8SPaolo Bonzini } 47980b4ecc8SPaolo Bonzini 48080b4ecc8SPaolo Bonzini entry->updated = true; 48180b4ecc8SPaolo Bonzini } 48280b4ecc8SPaolo Bonzini 48380b4ecc8SPaolo Bonzini set_entry_value(entry, offset, val); 48480b4ecc8SPaolo Bonzini 48580b4ecc8SPaolo Bonzini if (offset == PCI_MSIX_ENTRY_VECTOR_CTRL) { 48680b4ecc8SPaolo Bonzini if (msix->enabled && !(val & PCI_MSIX_ENTRY_CTRL_MASKBIT)) { 48780b4ecc8SPaolo Bonzini xen_pt_msix_update_one(s, entry_nr); 48880b4ecc8SPaolo Bonzini } 48980b4ecc8SPaolo Bonzini } 49080b4ecc8SPaolo Bonzini } 49180b4ecc8SPaolo Bonzini 49280b4ecc8SPaolo Bonzini static uint64_t pci_msix_read(void *opaque, hwaddr addr, 49380b4ecc8SPaolo Bonzini unsigned size) 49480b4ecc8SPaolo Bonzini { 49580b4ecc8SPaolo Bonzini XenPCIPassthroughState *s = opaque; 49680b4ecc8SPaolo Bonzini XenPTMSIX *msix = s->msix; 49780b4ecc8SPaolo Bonzini int entry_nr, offset; 49880b4ecc8SPaolo Bonzini 49980b4ecc8SPaolo Bonzini entry_nr = addr / PCI_MSIX_ENTRY_SIZE; 50080b4ecc8SPaolo Bonzini if (entry_nr < 0) { 50180b4ecc8SPaolo Bonzini XEN_PT_ERR(&s->dev, "asked MSI-X entry '%i' invalid!\n", entry_nr); 50280b4ecc8SPaolo Bonzini return 0; 50380b4ecc8SPaolo Bonzini } 50480b4ecc8SPaolo Bonzini 50580b4ecc8SPaolo Bonzini offset = addr % PCI_MSIX_ENTRY_SIZE; 50680b4ecc8SPaolo Bonzini 50780b4ecc8SPaolo Bonzini if (addr < msix->total_entries * PCI_MSIX_ENTRY_SIZE) { 50880b4ecc8SPaolo Bonzini return get_entry_value(&msix->msix_entry[entry_nr], offset); 50980b4ecc8SPaolo Bonzini } else { 51080b4ecc8SPaolo Bonzini /* Pending Bit Array (PBA) */ 51180b4ecc8SPaolo Bonzini return *(uint32_t *)(msix->phys_iomem_base + addr); 51280b4ecc8SPaolo Bonzini } 51380b4ecc8SPaolo Bonzini } 51480b4ecc8SPaolo Bonzini 51580b4ecc8SPaolo Bonzini static const MemoryRegionOps pci_msix_ops = { 51680b4ecc8SPaolo Bonzini .read = pci_msix_read, 51780b4ecc8SPaolo Bonzini .write = pci_msix_write, 51880b4ecc8SPaolo Bonzini .endianness = DEVICE_NATIVE_ENDIAN, 51980b4ecc8SPaolo Bonzini .valid = { 52080b4ecc8SPaolo Bonzini .min_access_size = 4, 52180b4ecc8SPaolo Bonzini .max_access_size = 4, 52280b4ecc8SPaolo Bonzini .unaligned = false, 52380b4ecc8SPaolo Bonzini }, 52480b4ecc8SPaolo Bonzini }; 52580b4ecc8SPaolo Bonzini 52680b4ecc8SPaolo Bonzini int xen_pt_msix_init(XenPCIPassthroughState *s, uint32_t base) 52780b4ecc8SPaolo Bonzini { 52880b4ecc8SPaolo Bonzini uint8_t id = 0; 52980b4ecc8SPaolo Bonzini uint16_t control = 0; 53080b4ecc8SPaolo Bonzini uint32_t table_off = 0; 53180b4ecc8SPaolo Bonzini int i, total_entries, bar_index; 53280b4ecc8SPaolo Bonzini XenHostPCIDevice *hd = &s->real_device; 53380b4ecc8SPaolo Bonzini PCIDevice *d = &s->dev; 53480b4ecc8SPaolo Bonzini int fd = -1; 53580b4ecc8SPaolo Bonzini XenPTMSIX *msix = NULL; 53680b4ecc8SPaolo Bonzini int rc = 0; 53780b4ecc8SPaolo Bonzini 53880b4ecc8SPaolo Bonzini rc = xen_host_pci_get_byte(hd, base + PCI_CAP_LIST_ID, &id); 53980b4ecc8SPaolo Bonzini if (rc) { 54080b4ecc8SPaolo Bonzini return rc; 54180b4ecc8SPaolo Bonzini } 54280b4ecc8SPaolo Bonzini 54380b4ecc8SPaolo Bonzini if (id != PCI_CAP_ID_MSIX) { 54480b4ecc8SPaolo Bonzini XEN_PT_ERR(d, "Invalid id %#x base %#x\n", id, base); 54580b4ecc8SPaolo Bonzini return -1; 54680b4ecc8SPaolo Bonzini } 54780b4ecc8SPaolo Bonzini 54880b4ecc8SPaolo Bonzini xen_host_pci_get_word(hd, base + PCI_MSIX_FLAGS, &control); 54980b4ecc8SPaolo Bonzini total_entries = control & PCI_MSIX_FLAGS_QSIZE; 55080b4ecc8SPaolo Bonzini total_entries += 1; 55180b4ecc8SPaolo Bonzini 55280b4ecc8SPaolo Bonzini s->msix = g_malloc0(sizeof (XenPTMSIX) 55380b4ecc8SPaolo Bonzini + total_entries * sizeof (XenPTMSIXEntry)); 55480b4ecc8SPaolo Bonzini msix = s->msix; 55580b4ecc8SPaolo Bonzini 55680b4ecc8SPaolo Bonzini msix->total_entries = total_entries; 55780b4ecc8SPaolo Bonzini for (i = 0; i < total_entries; i++) { 55880b4ecc8SPaolo Bonzini msix->msix_entry[i].pirq = XEN_PT_UNASSIGNED_PIRQ; 55980b4ecc8SPaolo Bonzini } 56080b4ecc8SPaolo Bonzini 56122fc860bSPaolo Bonzini memory_region_init_io(&msix->mmio, OBJECT(s), &pci_msix_ops, 56222fc860bSPaolo Bonzini s, "xen-pci-pt-msix", 56380b4ecc8SPaolo Bonzini (total_entries * PCI_MSIX_ENTRY_SIZE 56480b4ecc8SPaolo Bonzini + XC_PAGE_SIZE - 1) 56580b4ecc8SPaolo Bonzini & XC_PAGE_MASK); 56680b4ecc8SPaolo Bonzini 56780b4ecc8SPaolo Bonzini xen_host_pci_get_long(hd, base + PCI_MSIX_TABLE, &table_off); 56880b4ecc8SPaolo Bonzini bar_index = msix->bar_index = table_off & PCI_MSIX_FLAGS_BIRMASK; 56980b4ecc8SPaolo Bonzini table_off = table_off & ~PCI_MSIX_FLAGS_BIRMASK; 57080b4ecc8SPaolo Bonzini msix->table_base = s->real_device.io_regions[bar_index].base_addr; 57180b4ecc8SPaolo Bonzini XEN_PT_LOG(d, "get MSI-X table BAR base 0x%"PRIx64"\n", msix->table_base); 57280b4ecc8SPaolo Bonzini 57380b4ecc8SPaolo Bonzini fd = open("/dev/mem", O_RDWR); 57480b4ecc8SPaolo Bonzini if (fd == -1) { 57580b4ecc8SPaolo Bonzini rc = -errno; 57680b4ecc8SPaolo Bonzini XEN_PT_ERR(d, "Can't open /dev/mem: %s\n", strerror(errno)); 57780b4ecc8SPaolo Bonzini goto error_out; 57880b4ecc8SPaolo Bonzini } 57980b4ecc8SPaolo Bonzini XEN_PT_LOG(d, "table_off = %#x, total_entries = %d\n", 58080b4ecc8SPaolo Bonzini table_off, total_entries); 58180b4ecc8SPaolo Bonzini msix->table_offset_adjust = table_off & 0x0fff; 58280b4ecc8SPaolo Bonzini msix->phys_iomem_base = 58380b4ecc8SPaolo Bonzini mmap(NULL, 58480b4ecc8SPaolo Bonzini total_entries * PCI_MSIX_ENTRY_SIZE + msix->table_offset_adjust, 58580b4ecc8SPaolo Bonzini PROT_READ, 58680b4ecc8SPaolo Bonzini MAP_SHARED | MAP_LOCKED, 58780b4ecc8SPaolo Bonzini fd, 58880b4ecc8SPaolo Bonzini msix->table_base + table_off - msix->table_offset_adjust); 58980b4ecc8SPaolo Bonzini close(fd); 59080b4ecc8SPaolo Bonzini if (msix->phys_iomem_base == MAP_FAILED) { 59180b4ecc8SPaolo Bonzini rc = -errno; 59280b4ecc8SPaolo Bonzini XEN_PT_ERR(d, "Can't map physical MSI-X table: %s\n", strerror(errno)); 59380b4ecc8SPaolo Bonzini goto error_out; 59480b4ecc8SPaolo Bonzini } 59580b4ecc8SPaolo Bonzini msix->phys_iomem_base = (char *)msix->phys_iomem_base 59680b4ecc8SPaolo Bonzini + msix->table_offset_adjust; 59780b4ecc8SPaolo Bonzini 59880b4ecc8SPaolo Bonzini XEN_PT_LOG(d, "mapping physical MSI-X table to %p\n", 59980b4ecc8SPaolo Bonzini msix->phys_iomem_base); 60080b4ecc8SPaolo Bonzini 60180b4ecc8SPaolo Bonzini memory_region_add_subregion_overlap(&s->bar[bar_index], table_off, 60280b4ecc8SPaolo Bonzini &msix->mmio, 60380b4ecc8SPaolo Bonzini 2); /* Priority: pci default + 1 */ 60480b4ecc8SPaolo Bonzini 60580b4ecc8SPaolo Bonzini return 0; 60680b4ecc8SPaolo Bonzini 60780b4ecc8SPaolo Bonzini error_out: 60880b4ecc8SPaolo Bonzini g_free(s->msix); 60980b4ecc8SPaolo Bonzini s->msix = NULL; 61080b4ecc8SPaolo Bonzini return rc; 61180b4ecc8SPaolo Bonzini } 61280b4ecc8SPaolo Bonzini 61380b4ecc8SPaolo Bonzini void xen_pt_msix_delete(XenPCIPassthroughState *s) 61480b4ecc8SPaolo Bonzini { 61580b4ecc8SPaolo Bonzini XenPTMSIX *msix = s->msix; 61680b4ecc8SPaolo Bonzini 61780b4ecc8SPaolo Bonzini if (!msix) { 61880b4ecc8SPaolo Bonzini return; 61980b4ecc8SPaolo Bonzini } 62080b4ecc8SPaolo Bonzini 62180b4ecc8SPaolo Bonzini /* unmap the MSI-X memory mapped register area */ 62280b4ecc8SPaolo Bonzini if (msix->phys_iomem_base) { 62380b4ecc8SPaolo Bonzini XEN_PT_LOG(&s->dev, "unmapping physical MSI-X table from %p\n", 62480b4ecc8SPaolo Bonzini msix->phys_iomem_base); 62580b4ecc8SPaolo Bonzini munmap(msix->phys_iomem_base, msix->total_entries * PCI_MSIX_ENTRY_SIZE 62680b4ecc8SPaolo Bonzini + msix->table_offset_adjust); 62780b4ecc8SPaolo Bonzini } 62880b4ecc8SPaolo Bonzini 62980b4ecc8SPaolo Bonzini memory_region_del_subregion(&s->bar[msix->bar_index], &msix->mmio); 63080b4ecc8SPaolo Bonzini 63180b4ecc8SPaolo Bonzini g_free(s->msix); 63280b4ecc8SPaolo Bonzini s->msix = NULL; 63380b4ecc8SPaolo Bonzini } 634