15ca02815Sjsg // SPDX-License-Identifier: MIT
25ca02815Sjsg /*
35ca02815Sjsg * Copyright © 2019 Intel Corporation
45ca02815Sjsg */
55ca02815Sjsg
65ca02815Sjsg #include "i915_drv.h"
71bb76ff1Sjsg #include "i915_pci.h"
81bb76ff1Sjsg #include "i915_reg.h"
95ca02815Sjsg #include "intel_memory_region.h"
101bb76ff1Sjsg #include "intel_pci_config.h"
115ca02815Sjsg #include "intel_region_lmem.h"
125ca02815Sjsg #include "intel_region_ttm.h"
135ca02815Sjsg #include "gem/i915_gem_lmem.h"
145ca02815Sjsg #include "gem/i915_gem_region.h"
155ca02815Sjsg #include "gem/i915_gem_ttm.h"
165ca02815Sjsg #include "gt/intel_gt.h"
171bb76ff1Sjsg #include "gt/intel_gt_mcr.h"
181bb76ff1Sjsg #include "gt/intel_gt_regs.h"
195ca02815Sjsg
201bb76ff1Sjsg #ifdef CONFIG_64BIT
_release_bars(struct pci_dev * pdev)211bb76ff1Sjsg static void _release_bars(struct pci_dev *pdev)
225ca02815Sjsg {
235ca02815Sjsg STUB();
245ca02815Sjsg #ifdef notyet
251bb76ff1Sjsg int resno;
265ca02815Sjsg
271bb76ff1Sjsg for (resno = PCI_STD_RESOURCES; resno < PCI_STD_RESOURCE_END; resno++) {
281bb76ff1Sjsg if (pci_resource_len(pdev, resno))
291bb76ff1Sjsg pci_release_resource(pdev, resno);
301bb76ff1Sjsg }
315ca02815Sjsg #endif
325ca02815Sjsg }
335ca02815Sjsg
345ca02815Sjsg static void
_resize_bar(struct drm_i915_private * i915,int resno,resource_size_t size)351bb76ff1Sjsg _resize_bar(struct drm_i915_private *i915, int resno, resource_size_t size)
361bb76ff1Sjsg {
371bb76ff1Sjsg STUB();
381bb76ff1Sjsg #ifdef notyet
391bb76ff1Sjsg struct pci_dev *pdev = i915->drm.pdev;
401bb76ff1Sjsg int bar_size = pci_rebar_bytes_to_size(size);
411bb76ff1Sjsg int ret;
421bb76ff1Sjsg
431bb76ff1Sjsg _release_bars(pdev);
441bb76ff1Sjsg
451bb76ff1Sjsg ret = pci_resize_resource(pdev, resno, bar_size);
461bb76ff1Sjsg if (ret) {
471bb76ff1Sjsg drm_info(&i915->drm, "Failed to resize BAR%d to %dM (%pe)\n",
481bb76ff1Sjsg resno, 1 << bar_size, ERR_PTR(ret));
491bb76ff1Sjsg return;
501bb76ff1Sjsg }
511bb76ff1Sjsg
521bb76ff1Sjsg drm_info(&i915->drm, "BAR%d resized to %dM\n", resno, 1 << bar_size);
531bb76ff1Sjsg #endif
541bb76ff1Sjsg }
551bb76ff1Sjsg
i915_resize_lmem_bar(struct drm_i915_private * i915,resource_size_t lmem_size)561bb76ff1Sjsg static void i915_resize_lmem_bar(struct drm_i915_private *i915, resource_size_t lmem_size)
571bb76ff1Sjsg {
581bb76ff1Sjsg STUB();
591bb76ff1Sjsg #ifdef notyet
601bb76ff1Sjsg struct pci_dev *pdev = i915->drm.pdev;
611bb76ff1Sjsg struct pci_bus *root = pdev->bus;
621bb76ff1Sjsg struct resource *root_res;
631bb76ff1Sjsg resource_size_t rebar_size;
641bb76ff1Sjsg resource_size_t current_size;
65f005ef32Sjsg intel_wakeref_t wakeref;
661bb76ff1Sjsg u32 pci_cmd;
671bb76ff1Sjsg int i;
681bb76ff1Sjsg
691bb76ff1Sjsg current_size = roundup_pow_of_two(pci_resource_len(pdev, GEN12_LMEM_BAR));
701bb76ff1Sjsg
711bb76ff1Sjsg if (i915->params.lmem_bar_size) {
721bb76ff1Sjsg u32 bar_sizes;
731bb76ff1Sjsg
741bb76ff1Sjsg rebar_size = i915->params.lmem_bar_size *
751bb76ff1Sjsg (resource_size_t)SZ_1M;
761bb76ff1Sjsg bar_sizes = pci_rebar_get_possible_sizes(pdev, GEN12_LMEM_BAR);
771bb76ff1Sjsg
781bb76ff1Sjsg if (rebar_size == current_size)
791bb76ff1Sjsg return;
801bb76ff1Sjsg
811bb76ff1Sjsg if (!(bar_sizes & BIT(pci_rebar_bytes_to_size(rebar_size))) ||
821bb76ff1Sjsg rebar_size >= roundup_pow_of_two(lmem_size)) {
831bb76ff1Sjsg rebar_size = lmem_size;
841bb76ff1Sjsg
851bb76ff1Sjsg drm_info(&i915->drm,
861bb76ff1Sjsg "Given bar size is not within supported size, setting it to default: %llu\n",
871bb76ff1Sjsg (u64)lmem_size >> 20);
881bb76ff1Sjsg }
891bb76ff1Sjsg } else {
901bb76ff1Sjsg rebar_size = current_size;
911bb76ff1Sjsg
921bb76ff1Sjsg if (rebar_size != roundup_pow_of_two(lmem_size))
931bb76ff1Sjsg rebar_size = lmem_size;
941bb76ff1Sjsg else
951bb76ff1Sjsg return;
961bb76ff1Sjsg }
971bb76ff1Sjsg
981bb76ff1Sjsg /* Find out if root bus contains 64bit memory addressing */
991bb76ff1Sjsg while (root->parent)
1001bb76ff1Sjsg root = root->parent;
1011bb76ff1Sjsg
1021bb76ff1Sjsg pci_bus_for_each_resource(root, root_res, i) {
1031bb76ff1Sjsg if (root_res && root_res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1041bb76ff1Sjsg root_res->start > 0x100000000ull)
1051bb76ff1Sjsg break;
1061bb76ff1Sjsg }
1071bb76ff1Sjsg
1081bb76ff1Sjsg /* pci_resize_resource will fail anyways */
1091bb76ff1Sjsg if (!root_res) {
1101bb76ff1Sjsg drm_info(&i915->drm, "Can't resize LMEM BAR - platform support is missing\n");
1111bb76ff1Sjsg return;
1121bb76ff1Sjsg }
1131bb76ff1Sjsg
114f005ef32Sjsg /*
115f005ef32Sjsg * Releasing forcewake during BAR resizing results in later forcewake
116f005ef32Sjsg * ack timeouts and former can happen any time - it is asynchronous.
117f005ef32Sjsg * Grabbing all forcewakes prevents it.
118f005ef32Sjsg */
119f005ef32Sjsg with_intel_runtime_pm(i915->uncore.rpm, wakeref) {
120f005ef32Sjsg intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
121f005ef32Sjsg
1221bb76ff1Sjsg /* First disable PCI memory decoding references */
1231bb76ff1Sjsg pci_read_config_dword(pdev, PCI_COMMAND, &pci_cmd);
1241bb76ff1Sjsg pci_write_config_dword(pdev, PCI_COMMAND,
1251bb76ff1Sjsg pci_cmd & ~PCI_COMMAND_MEMORY);
1261bb76ff1Sjsg
1271bb76ff1Sjsg _resize_bar(i915, GEN12_LMEM_BAR, rebar_size);
1281bb76ff1Sjsg
1291bb76ff1Sjsg pci_assign_unassigned_bus_resources(pdev->bus);
1301bb76ff1Sjsg pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd);
131f005ef32Sjsg intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
132f005ef32Sjsg }
1331bb76ff1Sjsg #endif
1341bb76ff1Sjsg }
1351bb76ff1Sjsg #else
i915_resize_lmem_bar(struct drm_i915_private * i915,resource_size_t lmem_size)1361bb76ff1Sjsg static void i915_resize_lmem_bar(struct drm_i915_private *i915, resource_size_t lmem_size) {}
1371bb76ff1Sjsg #endif
1381bb76ff1Sjsg
1391bb76ff1Sjsg static int
region_lmem_release(struct intel_memory_region * mem)1405ca02815Sjsg region_lmem_release(struct intel_memory_region *mem)
1415ca02815Sjsg {
1421bb76ff1Sjsg int ret;
1431bb76ff1Sjsg
1441bb76ff1Sjsg ret = intel_region_ttm_fini(mem);
1455ca02815Sjsg STUB();
1465ca02815Sjsg #ifdef notyet
1475ca02815Sjsg io_mapping_fini(&mem->iomap);
1485ca02815Sjsg #endif
1491bb76ff1Sjsg
1501bb76ff1Sjsg return ret;
1515ca02815Sjsg }
1525ca02815Sjsg
1535ca02815Sjsg static int
region_lmem_init(struct intel_memory_region * mem)1545ca02815Sjsg region_lmem_init(struct intel_memory_region *mem)
1555ca02815Sjsg {
1565ca02815Sjsg int ret;
1575ca02815Sjsg
158*56f0ffbeSjsg #ifdef __linux__
1595ca02815Sjsg if (!io_mapping_init_wc(&mem->iomap,
1605ca02815Sjsg mem->io_start,
1611bb76ff1Sjsg mem->io_size))
1621bb76ff1Sjsg return -EIO;
163*56f0ffbeSjsg #else
164*56f0ffbeSjsg struct drm_i915_private *i915 = mem->i915;
165*56f0ffbeSjsg paddr_t start, end;
166*56f0ffbeSjsg struct vm_page *pgs;
167*56f0ffbeSjsg int i;
168*56f0ffbeSjsg bus_space_handle_t bsh;
169*56f0ffbeSjsg
170*56f0ffbeSjsg start = atop(mem->io_start);
171*56f0ffbeSjsg end = start + atop(mem->io_size);
172*56f0ffbeSjsg uvm_page_physload(start, end, start, end, PHYSLOAD_DEVICE);
173*56f0ffbeSjsg
174*56f0ffbeSjsg pgs = PHYS_TO_VM_PAGE(mem->io_start);
175*56f0ffbeSjsg for (i = 0; i < atop(mem->io_size); i++)
176*56f0ffbeSjsg atomic_setbits_int(&(pgs[i].pg_flags), PG_PMAP_WC);
177*56f0ffbeSjsg
178*56f0ffbeSjsg if (bus_space_map(i915->bst, mem->io_start, mem->io_size,
179*56f0ffbeSjsg BUS_SPACE_MAP_LINEAR | BUS_SPACE_MAP_PREFETCHABLE, &bsh))
180*56f0ffbeSjsg panic("can't map lmem");
181*56f0ffbeSjsg
182*56f0ffbeSjsg mem->iomap.base = mem->io_start;
183*56f0ffbeSjsg mem->iomap.size = mem->io_size;
184*56f0ffbeSjsg mem->iomap.iomem = bus_space_vaddr(i915->bst, bsh);
185*56f0ffbeSjsg #endif
1865ca02815Sjsg
1875ca02815Sjsg ret = intel_region_ttm_init(mem);
1885ca02815Sjsg if (ret)
1895ca02815Sjsg goto out_no_buddy;
1905ca02815Sjsg
1915ca02815Sjsg return 0;
1925ca02815Sjsg
1935ca02815Sjsg out_no_buddy:
194*56f0ffbeSjsg #ifdef __linux__
1955ca02815Sjsg io_mapping_fini(&mem->iomap);
196*56f0ffbeSjsg #endif
1975ca02815Sjsg
1985ca02815Sjsg return ret;
1995ca02815Sjsg }
2005ca02815Sjsg
2015ca02815Sjsg static const struct intel_memory_region_ops intel_region_lmem_ops = {
2025ca02815Sjsg .init = region_lmem_init,
2035ca02815Sjsg .release = region_lmem_release,
2045ca02815Sjsg .init_object = __i915_gem_ttm_object_init,
2055ca02815Sjsg };
2065ca02815Sjsg
get_legacy_lowmem_region(struct intel_uncore * uncore,u64 * start,u32 * size)2075ca02815Sjsg static bool get_legacy_lowmem_region(struct intel_uncore *uncore,
2085ca02815Sjsg u64 *start, u32 *size)
2095ca02815Sjsg {
210f005ef32Sjsg if (!IS_DG1(uncore->i915))
2115ca02815Sjsg return false;
2125ca02815Sjsg
2135ca02815Sjsg *start = 0;
2145ca02815Sjsg *size = SZ_1M;
2155ca02815Sjsg
2165ca02815Sjsg drm_dbg(&uncore->i915->drm, "LMEM: reserved legacy low-memory [0x%llx-0x%llx]\n",
2175ca02815Sjsg *start, *start + *size);
2185ca02815Sjsg
2195ca02815Sjsg return true;
2205ca02815Sjsg }
2215ca02815Sjsg
reserve_lowmem_region(struct intel_uncore * uncore,struct intel_memory_region * mem)2225ca02815Sjsg static int reserve_lowmem_region(struct intel_uncore *uncore,
2235ca02815Sjsg struct intel_memory_region *mem)
2245ca02815Sjsg {
2255ca02815Sjsg u64 reserve_start;
2265ca02815Sjsg u32 reserve_size;
2275ca02815Sjsg int ret;
2285ca02815Sjsg
2295ca02815Sjsg if (!get_legacy_lowmem_region(uncore, &reserve_start, &reserve_size))
2305ca02815Sjsg return 0;
2315ca02815Sjsg
2325ca02815Sjsg ret = intel_memory_region_reserve(mem, reserve_start, reserve_size);
2335ca02815Sjsg if (ret)
2345ca02815Sjsg drm_err(&uncore->i915->drm, "LMEM: reserving low memory region failed\n");
2355ca02815Sjsg
2365ca02815Sjsg return ret;
2375ca02815Sjsg }
2385ca02815Sjsg
setup_lmem(struct intel_gt * gt)2395ca02815Sjsg static struct intel_memory_region *setup_lmem(struct intel_gt *gt)
2405ca02815Sjsg {
2415ca02815Sjsg struct drm_i915_private *i915 = gt->i915;
2425ca02815Sjsg struct intel_uncore *uncore = gt->uncore;
2435ca02815Sjsg struct pci_dev *pdev = i915->drm.pdev;
2445ca02815Sjsg struct intel_memory_region *mem;
2451bb76ff1Sjsg resource_size_t min_page_size;
2465ca02815Sjsg resource_size_t io_start;
2471bb76ff1Sjsg resource_size_t io_size;
2485ca02815Sjsg resource_size_t lmem_size;
2495ca02815Sjsg int err;
2505ca02815Sjsg
2515ca02815Sjsg if (!IS_DGFX(i915))
2525ca02815Sjsg return ERR_PTR(-ENODEV);
2535ca02815Sjsg
2541bb76ff1Sjsg #ifdef notyet
2551bb76ff1Sjsg if (!i915_pci_resource_valid(pdev, GEN12_LMEM_BAR))
2561bb76ff1Sjsg return ERR_PTR(-ENXIO);
2575ca02815Sjsg #endif
2585ca02815Sjsg
2591bb76ff1Sjsg if (HAS_FLAT_CCS(i915)) {
2601bb76ff1Sjsg resource_size_t lmem_range;
2611bb76ff1Sjsg u64 tile_stolen, flat_ccs_base;
2621bb76ff1Sjsg
263f005ef32Sjsg lmem_range = intel_gt_mcr_read_any(to_gt(i915), XEHP_TILE0_ADDR_RANGE) & 0xFFFF;
2641bb76ff1Sjsg lmem_size = lmem_range >> XEHP_TILE_LMEM_RANGE_SHIFT;
2651bb76ff1Sjsg lmem_size *= SZ_1G;
2661bb76ff1Sjsg
2671bb76ff1Sjsg flat_ccs_base = intel_gt_mcr_read_any(gt, XEHP_FLAT_CCS_BASE_ADDR);
2681bb76ff1Sjsg flat_ccs_base = (flat_ccs_base >> XEHP_CCS_BASE_SHIFT) * SZ_64K;
2691bb76ff1Sjsg
2701bb76ff1Sjsg if (GEM_WARN_ON(lmem_size < flat_ccs_base))
2711bb76ff1Sjsg return ERR_PTR(-EIO);
2721bb76ff1Sjsg
2731bb76ff1Sjsg tile_stolen = lmem_size - flat_ccs_base;
2741bb76ff1Sjsg
2751bb76ff1Sjsg /* If the FLAT_CCS_BASE_ADDR register is not populated, flag an error */
2761bb76ff1Sjsg if (tile_stolen == lmem_size)
2771bb76ff1Sjsg drm_err(&i915->drm,
2781bb76ff1Sjsg "CCS_BASE_ADDR register did not have expected value\n");
2791bb76ff1Sjsg
2801bb76ff1Sjsg lmem_size -= tile_stolen;
2811bb76ff1Sjsg } else {
2821bb76ff1Sjsg /* Stolen starts from GSMBASE without CCS */
2831bb76ff1Sjsg lmem_size = intel_uncore_read64(&i915->uncore, GEN12_GSMBASE);
2841bb76ff1Sjsg }
2851bb76ff1Sjsg
2861bb76ff1Sjsg i915_resize_lmem_bar(i915, lmem_size);
2871bb76ff1Sjsg
2881bb76ff1Sjsg if (i915->params.lmem_size > 0) {
2891bb76ff1Sjsg lmem_size = min_t(resource_size_t, lmem_size,
2901bb76ff1Sjsg mul_u32_u32(i915->params.lmem_size, SZ_1M));
2911bb76ff1Sjsg }
2921bb76ff1Sjsg
2931bb76ff1Sjsg #ifdef __linux__
2941bb76ff1Sjsg io_start = pci_resource_start(pdev, GEN12_LMEM_BAR);
2951bb76ff1Sjsg io_size = min(pci_resource_len(pdev, GEN12_LMEM_BAR), lmem_size);
2961bb76ff1Sjsg #else
2971bb76ff1Sjsg {
2981bb76ff1Sjsg pcireg_t type;
2991bb76ff1Sjsg bus_size_t len;
3001bb76ff1Sjsg
3011bb76ff1Sjsg type = pci_mapreg_type(i915->pc, i915->tag,
3021bb76ff1Sjsg 0x10 + (4 * GEN12_LMEM_BAR));
3031bb76ff1Sjsg err = -pci_mapreg_info(i915->pc, i915->tag,
3041bb76ff1Sjsg 0x10 + (4 * GEN12_LMEM_BAR), type, &io_start, &len, NULL);
3051bb76ff1Sjsg io_size = min(len, lmem_size);
3061bb76ff1Sjsg }
3071bb76ff1Sjsg #endif
3081bb76ff1Sjsg if (!io_size)
3091bb76ff1Sjsg return ERR_PTR(-EIO);
3101bb76ff1Sjsg
3111bb76ff1Sjsg min_page_size = HAS_64K_PAGES(i915) ? I915_GTT_PAGE_SIZE_64K :
3121bb76ff1Sjsg I915_GTT_PAGE_SIZE_4K;
3135ca02815Sjsg mem = intel_memory_region_create(i915,
3145ca02815Sjsg 0,
3155ca02815Sjsg lmem_size,
3161bb76ff1Sjsg min_page_size,
3175ca02815Sjsg io_start,
3181bb76ff1Sjsg io_size,
3195ca02815Sjsg INTEL_MEMORY_LOCAL,
3205ca02815Sjsg 0,
3215ca02815Sjsg &intel_region_lmem_ops);
3225ca02815Sjsg if (IS_ERR(mem))
3235ca02815Sjsg return mem;
3245ca02815Sjsg
3255ca02815Sjsg err = reserve_lowmem_region(uncore, mem);
3265ca02815Sjsg if (err)
3275ca02815Sjsg goto err_region_put;
3285ca02815Sjsg
3295ca02815Sjsg drm_dbg(&i915->drm, "Local memory: %pR\n", &mem->region);
3305ca02815Sjsg drm_dbg(&i915->drm, "Local memory IO start: %pa\n",
3315ca02815Sjsg &mem->io_start);
3321bb76ff1Sjsg drm_info(&i915->drm, "Local memory IO size: %pa\n",
3331bb76ff1Sjsg &mem->io_size);
3345ca02815Sjsg drm_info(&i915->drm, "Local memory available: %pa\n",
3355ca02815Sjsg &lmem_size);
3365ca02815Sjsg
3371bb76ff1Sjsg if (io_size < lmem_size)
3381bb76ff1Sjsg drm_info(&i915->drm, "Using a reduced BAR size of %lluMiB. Consider enabling 'Resizable BAR' or similar, if available in the BIOS.\n",
3391bb76ff1Sjsg (u64)io_size >> 20);
3401bb76ff1Sjsg
3415ca02815Sjsg return mem;
3425ca02815Sjsg
3435ca02815Sjsg err_region_put:
3441bb76ff1Sjsg intel_memory_region_destroy(mem);
3455ca02815Sjsg return ERR_PTR(err);
3465ca02815Sjsg }
3475ca02815Sjsg
intel_gt_setup_lmem(struct intel_gt * gt)3485ca02815Sjsg struct intel_memory_region *intel_gt_setup_lmem(struct intel_gt *gt)
3495ca02815Sjsg {
3505ca02815Sjsg return setup_lmem(gt);
3515ca02815Sjsg }
352