1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2b8ef9172STejun Heo /*
3b8ef9172STejun Heo * NUMA emulation
4b8ef9172STejun Heo */
5b8ef9172STejun Heo #include <linux/kernel.h>
6b8ef9172STejun Heo #include <linux/errno.h>
7b8ef9172STejun Heo #include <linux/topology.h>
8b8ef9172STejun Heo #include <linux/memblock.h>
9b8ef9172STejun Heo #include <asm/dma.h>
10b8ef9172STejun Heo
11b8ef9172STejun Heo #include "numa_internal.h"
12b8ef9172STejun Heo
13148f9bb8SPaul Gortmaker static int emu_nid_to_phys[MAX_NUMNODES];
14b8ef9172STejun Heo static char *emu_cmdline __initdata;
15b8ef9172STejun Heo
numa_emu_cmdline(char * str)162dd57d34SDan Williams int __init numa_emu_cmdline(char *str)
17b8ef9172STejun Heo {
18b8ef9172STejun Heo emu_cmdline = str;
192dd57d34SDan Williams return 0;
20b8ef9172STejun Heo }
21b8ef9172STejun Heo
emu_find_memblk_by_nid(int nid,const struct numa_meminfo * mi)22b8ef9172STejun Heo static int __init emu_find_memblk_by_nid(int nid, const struct numa_meminfo *mi)
23b8ef9172STejun Heo {
24b8ef9172STejun Heo int i;
25b8ef9172STejun Heo
26b8ef9172STejun Heo for (i = 0; i < mi->nr_blks; i++)
27b8ef9172STejun Heo if (mi->blk[i].nid == nid)
28b8ef9172STejun Heo return i;
29b8ef9172STejun Heo return -ENOENT;
30b8ef9172STejun Heo }
31b8ef9172STejun Heo
mem_hole_size(u64 start,u64 end)32e37aade3SJiri Kosina static u64 __init mem_hole_size(u64 start, u64 end)
33474b881bSTejun Heo {
34474b881bSTejun Heo unsigned long start_pfn = PFN_UP(start);
35474b881bSTejun Heo unsigned long end_pfn = PFN_DOWN(end);
36474b881bSTejun Heo
37474b881bSTejun Heo if (start_pfn < end_pfn)
38474b881bSTejun Heo return PFN_PHYS(absent_pages_in_range(start_pfn, end_pfn));
39474b881bSTejun Heo return 0;
40474b881bSTejun Heo }
41474b881bSTejun Heo
42b8ef9172STejun Heo /*
43b8ef9172STejun Heo * Sets up nid to range from @start to @end. The return value is -errno if
44b8ef9172STejun Heo * something went wrong, 0 otherwise.
45b8ef9172STejun Heo */
emu_setup_memblk(struct numa_meminfo * ei,struct numa_meminfo * pi,int nid,int phys_blk,u64 size)46b8ef9172STejun Heo static int __init emu_setup_memblk(struct numa_meminfo *ei,
47b8ef9172STejun Heo struct numa_meminfo *pi,
48b8ef9172STejun Heo int nid, int phys_blk, u64 size)
49b8ef9172STejun Heo {
50b8ef9172STejun Heo struct numa_memblk *eb = &ei->blk[ei->nr_blks];
51b8ef9172STejun Heo struct numa_memblk *pb = &pi->blk[phys_blk];
52b8ef9172STejun Heo
53b8ef9172STejun Heo if (ei->nr_blks >= NR_NODE_MEMBLKS) {
54b8ef9172STejun Heo pr_err("NUMA: Too many emulated memblks, failing emulation\n");
55b8ef9172STejun Heo return -EINVAL;
56b8ef9172STejun Heo }
57b8ef9172STejun Heo
58b8ef9172STejun Heo ei->nr_blks++;
59b8ef9172STejun Heo eb->start = pb->start;
60b8ef9172STejun Heo eb->end = pb->start + size;
61b8ef9172STejun Heo eb->nid = nid;
62b8ef9172STejun Heo
63b8ef9172STejun Heo if (emu_nid_to_phys[nid] == NUMA_NO_NODE)
643b6c62f3SDan Williams emu_nid_to_phys[nid] = pb->nid;
65b8ef9172STejun Heo
66b8ef9172STejun Heo pb->start += size;
67b8ef9172STejun Heo if (pb->start >= pb->end) {
68b8ef9172STejun Heo WARN_ON_ONCE(pb->start > pb->end);
69b8ef9172STejun Heo numa_remove_memblk_from(phys_blk, pi);
70b8ef9172STejun Heo }
71b8ef9172STejun Heo
72365811d6SBjorn Helgaas printk(KERN_INFO "Faking node %d at [mem %#018Lx-%#018Lx] (%LuMB)\n",
73365811d6SBjorn Helgaas nid, eb->start, eb->end - 1, (eb->end - eb->start) >> 20);
74b8ef9172STejun Heo return 0;
75b8ef9172STejun Heo }
76b8ef9172STejun Heo
77b8ef9172STejun Heo /*
78b8ef9172STejun Heo * Sets up nr_nodes fake nodes interleaved over physical nodes ranging from addr
79d80a9eb3SWei Yang * to max_addr.
80d80a9eb3SWei Yang *
81d80a9eb3SWei Yang * Returns zero on success or negative on error.
82b8ef9172STejun Heo */
split_nodes_interleave(struct numa_meminfo * ei,struct numa_meminfo * pi,u64 addr,u64 max_addr,int nr_nodes)83b8ef9172STejun Heo static int __init split_nodes_interleave(struct numa_meminfo *ei,
84b8ef9172STejun Heo struct numa_meminfo *pi,
85b8ef9172STejun Heo u64 addr, u64 max_addr, int nr_nodes)
86b8ef9172STejun Heo {
87d80a9eb3SWei Yang nodemask_t physnode_mask = numa_nodes_parsed;
88b8ef9172STejun Heo u64 size;
89b8ef9172STejun Heo int big;
90b8ef9172STejun Heo int nid = 0;
91b8ef9172STejun Heo int i, ret;
92b8ef9172STejun Heo
93b8ef9172STejun Heo if (nr_nodes <= 0)
94b8ef9172STejun Heo return -1;
95b8ef9172STejun Heo if (nr_nodes > MAX_NUMNODES) {
96b8ef9172STejun Heo pr_info("numa=fake=%d too large, reducing to %d\n",
97b8ef9172STejun Heo nr_nodes, MAX_NUMNODES);
98b8ef9172STejun Heo nr_nodes = MAX_NUMNODES;
99b8ef9172STejun Heo }
100b8ef9172STejun Heo
1011b7e03efSTejun Heo /*
1021b7e03efSTejun Heo * Calculate target node size. x86_32 freaks on __udivdi3() so do
1031b7e03efSTejun Heo * the division in ulong number of pages and convert back.
1041b7e03efSTejun Heo */
105474b881bSTejun Heo size = max_addr - addr - mem_hole_size(addr, max_addr);
1061b7e03efSTejun Heo size = PFN_PHYS((unsigned long)(size >> PAGE_SHIFT) / nr_nodes);
1071b7e03efSTejun Heo
108b8ef9172STejun Heo /*
109b8ef9172STejun Heo * Calculate the number of big nodes that can be allocated as a result
110b8ef9172STejun Heo * of consolidating the remainder.
111b8ef9172STejun Heo */
112b8ef9172STejun Heo big = ((size & ~FAKE_NODE_MIN_HASH_MASK) * nr_nodes) /
113b8ef9172STejun Heo FAKE_NODE_MIN_SIZE;
114b8ef9172STejun Heo
115b8ef9172STejun Heo size &= FAKE_NODE_MIN_HASH_MASK;
116b8ef9172STejun Heo if (!size) {
117b8ef9172STejun Heo pr_err("Not enough memory for each node. "
118b8ef9172STejun Heo "NUMA emulation disabled.\n");
119b8ef9172STejun Heo return -1;
120b8ef9172STejun Heo }
121b8ef9172STejun Heo
122b8ef9172STejun Heo /*
123b8ef9172STejun Heo * Continue to fill physical nodes with fake nodes until there is no
124b8ef9172STejun Heo * memory left on any of them.
125b8ef9172STejun Heo */
126c2a911d3SYury Norov while (!nodes_empty(physnode_mask)) {
127b8ef9172STejun Heo for_each_node_mask(i, physnode_mask) {
128b8ef9172STejun Heo u64 dma32_end = PFN_PHYS(MAX_DMA32_PFN);
129b8ef9172STejun Heo u64 start, limit, end;
130b8ef9172STejun Heo int phys_blk;
131b8ef9172STejun Heo
132b8ef9172STejun Heo phys_blk = emu_find_memblk_by_nid(i, pi);
133b8ef9172STejun Heo if (phys_blk < 0) {
134b8ef9172STejun Heo node_clear(i, physnode_mask);
135b8ef9172STejun Heo continue;
136b8ef9172STejun Heo }
137b8ef9172STejun Heo start = pi->blk[phys_blk].start;
138b8ef9172STejun Heo limit = pi->blk[phys_blk].end;
139b8ef9172STejun Heo end = start + size;
140b8ef9172STejun Heo
141b8ef9172STejun Heo if (nid < big)
142b8ef9172STejun Heo end += FAKE_NODE_MIN_SIZE;
143b8ef9172STejun Heo
144b8ef9172STejun Heo /*
145b8ef9172STejun Heo * Continue to add memory to this fake node if its
146b8ef9172STejun Heo * non-reserved memory is less than the per-node size.
147b8ef9172STejun Heo */
148474b881bSTejun Heo while (end - start - mem_hole_size(start, end) < size) {
149b8ef9172STejun Heo end += FAKE_NODE_MIN_SIZE;
150b8ef9172STejun Heo if (end > limit) {
151b8ef9172STejun Heo end = limit;
152b8ef9172STejun Heo break;
153b8ef9172STejun Heo }
154b8ef9172STejun Heo }
155b8ef9172STejun Heo
156b8ef9172STejun Heo /*
157b8ef9172STejun Heo * If there won't be at least FAKE_NODE_MIN_SIZE of
158b8ef9172STejun Heo * non-reserved memory in ZONE_DMA32 for the next node,
159b8ef9172STejun Heo * this one must extend to the boundary.
160b8ef9172STejun Heo */
161b8ef9172STejun Heo if (end < dma32_end && dma32_end - end -
162474b881bSTejun Heo mem_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE)
163b8ef9172STejun Heo end = dma32_end;
164b8ef9172STejun Heo
165b8ef9172STejun Heo /*
166b8ef9172STejun Heo * If there won't be enough non-reserved memory for the
167b8ef9172STejun Heo * next node, this one must extend to the end of the
168b8ef9172STejun Heo * physical node.
169b8ef9172STejun Heo */
170474b881bSTejun Heo if (limit - end - mem_hole_size(end, limit) < size)
171b8ef9172STejun Heo end = limit;
172b8ef9172STejun Heo
173b8ef9172STejun Heo ret = emu_setup_memblk(ei, pi, nid++ % nr_nodes,
174b8ef9172STejun Heo phys_blk,
175b8ef9172STejun Heo min(end, limit) - start);
176b8ef9172STejun Heo if (ret < 0)
177b8ef9172STejun Heo return ret;
178b8ef9172STejun Heo }
179b8ef9172STejun Heo }
180b8ef9172STejun Heo return 0;
181b8ef9172STejun Heo }
182b8ef9172STejun Heo
183b8ef9172STejun Heo /*
184b8ef9172STejun Heo * Returns the end address of a node so that there is at least `size' amount of
185b8ef9172STejun Heo * non-reserved memory or `max_addr' is reached.
186b8ef9172STejun Heo */
find_end_of_node(u64 start,u64 max_addr,u64 size)187b8ef9172STejun Heo static u64 __init find_end_of_node(u64 start, u64 max_addr, u64 size)
188b8ef9172STejun Heo {
189b8ef9172STejun Heo u64 end = start + size;
190b8ef9172STejun Heo
191474b881bSTejun Heo while (end - start - mem_hole_size(start, end) < size) {
192b8ef9172STejun Heo end += FAKE_NODE_MIN_SIZE;
193b8ef9172STejun Heo if (end > max_addr) {
194b8ef9172STejun Heo end = max_addr;
195b8ef9172STejun Heo break;
196b8ef9172STejun Heo }
197b8ef9172STejun Heo }
198b8ef9172STejun Heo return end;
199b8ef9172STejun Heo }
200b8ef9172STejun Heo
uniform_size(u64 max_addr,u64 base,u64 hole,int nr_nodes)201cc9aec03SDan Williams static u64 uniform_size(u64 max_addr, u64 base, u64 hole, int nr_nodes)
202cc9aec03SDan Williams {
203cc9aec03SDan Williams unsigned long max_pfn = PHYS_PFN(max_addr);
204cc9aec03SDan Williams unsigned long base_pfn = PHYS_PFN(base);
205cc9aec03SDan Williams unsigned long hole_pfns = PHYS_PFN(hole);
206cc9aec03SDan Williams
207cc9aec03SDan Williams return PFN_PHYS((max_pfn - base_pfn - hole_pfns) / nr_nodes);
208cc9aec03SDan Williams }
209cc9aec03SDan Williams
210b8ef9172STejun Heo /*
211b8ef9172STejun Heo * Sets up fake nodes of `size' interleaved over physical nodes ranging from
212d80a9eb3SWei Yang * `addr' to `max_addr'.
213d80a9eb3SWei Yang *
214d80a9eb3SWei Yang * Returns zero on success or negative on error.
215b8ef9172STejun Heo */
split_nodes_size_interleave_uniform(struct numa_meminfo * ei,struct numa_meminfo * pi,u64 addr,u64 max_addr,u64 size,int nr_nodes,struct numa_memblk * pblk,int nid)216cc9aec03SDan Williams static int __init split_nodes_size_interleave_uniform(struct numa_meminfo *ei,
217b8ef9172STejun Heo struct numa_meminfo *pi,
218cc9aec03SDan Williams u64 addr, u64 max_addr, u64 size,
219cc9aec03SDan Williams int nr_nodes, struct numa_memblk *pblk,
220cc9aec03SDan Williams int nid)
221b8ef9172STejun Heo {
222d80a9eb3SWei Yang nodemask_t physnode_mask = numa_nodes_parsed;
223cc9aec03SDan Williams int i, ret, uniform = 0;
224b8ef9172STejun Heo u64 min_size;
225b8ef9172STejun Heo
226cc9aec03SDan Williams if ((!size && !nr_nodes) || (nr_nodes && !pblk))
227b8ef9172STejun Heo return -1;
228cc9aec03SDan Williams
229b8ef9172STejun Heo /*
230cc9aec03SDan Williams * In the 'uniform' case split the passed in physical node by
231cc9aec03SDan Williams * nr_nodes, in the non-uniform case, ignore the passed in
232cc9aec03SDan Williams * physical block and try to create nodes of at least size
233cc9aec03SDan Williams * @size.
234cc9aec03SDan Williams *
235cc9aec03SDan Williams * In the uniform case, split the nodes strictly by physical
236cc9aec03SDan Williams * capacity, i.e. ignore holes. In the non-uniform case account
237cc9aec03SDan Williams * for holes and treat @size as a minimum floor.
238b8ef9172STejun Heo */
239cc9aec03SDan Williams if (!nr_nodes)
240cc9aec03SDan Williams nr_nodes = MAX_NUMNODES;
241cc9aec03SDan Williams else {
242cc9aec03SDan Williams nodes_clear(physnode_mask);
243cc9aec03SDan Williams node_set(pblk->nid, physnode_mask);
244cc9aec03SDan Williams uniform = 1;
245cc9aec03SDan Williams }
246cc9aec03SDan Williams
247cc9aec03SDan Williams if (uniform) {
248cc9aec03SDan Williams min_size = uniform_size(max_addr, addr, 0, nr_nodes);
249cc9aec03SDan Williams size = min_size;
250cc9aec03SDan Williams } else {
251cc9aec03SDan Williams /*
252cc9aec03SDan Williams * The limit on emulated nodes is MAX_NUMNODES, so the
253cc9aec03SDan Williams * size per node is increased accordingly if the
254cc9aec03SDan Williams * requested size is too small. This creates a uniform
255cc9aec03SDan Williams * distribution of node sizes across the entire machine
256cc9aec03SDan Williams * (but not necessarily over physical nodes).
257cc9aec03SDan Williams */
258cc9aec03SDan Williams min_size = uniform_size(max_addr, addr,
259cc9aec03SDan Williams mem_hole_size(addr, max_addr), nr_nodes);
260cc9aec03SDan Williams }
261cc9aec03SDan Williams min_size = ALIGN(max(min_size, FAKE_NODE_MIN_SIZE), FAKE_NODE_MIN_SIZE);
262b8ef9172STejun Heo if (size < min_size) {
263b8ef9172STejun Heo pr_err("Fake node size %LuMB too small, increasing to %LuMB\n",
264b8ef9172STejun Heo size >> 20, min_size >> 20);
265b8ef9172STejun Heo size = min_size;
266b8ef9172STejun Heo }
267cc9aec03SDan Williams size = ALIGN_DOWN(size, FAKE_NODE_MIN_SIZE);
268b8ef9172STejun Heo
269b8ef9172STejun Heo /*
270b8ef9172STejun Heo * Fill physical nodes with fake nodes of size until there is no memory
271b8ef9172STejun Heo * left on any of them.
272b8ef9172STejun Heo */
273c2a911d3SYury Norov while (!nodes_empty(physnode_mask)) {
274b8ef9172STejun Heo for_each_node_mask(i, physnode_mask) {
2751b7e03efSTejun Heo u64 dma32_end = PFN_PHYS(MAX_DMA32_PFN);
276b8ef9172STejun Heo u64 start, limit, end;
277b8ef9172STejun Heo int phys_blk;
278b8ef9172STejun Heo
279b8ef9172STejun Heo phys_blk = emu_find_memblk_by_nid(i, pi);
280b8ef9172STejun Heo if (phys_blk < 0) {
281b8ef9172STejun Heo node_clear(i, physnode_mask);
282b8ef9172STejun Heo continue;
283b8ef9172STejun Heo }
284cc9aec03SDan Williams
285b8ef9172STejun Heo start = pi->blk[phys_blk].start;
286b8ef9172STejun Heo limit = pi->blk[phys_blk].end;
287b8ef9172STejun Heo
288cc9aec03SDan Williams if (uniform)
289cc9aec03SDan Williams end = start + size;
290cc9aec03SDan Williams else
291b8ef9172STejun Heo end = find_end_of_node(start, limit, size);
292b8ef9172STejun Heo /*
293b8ef9172STejun Heo * If there won't be at least FAKE_NODE_MIN_SIZE of
294b8ef9172STejun Heo * non-reserved memory in ZONE_DMA32 for the next node,
295b8ef9172STejun Heo * this one must extend to the boundary.
296b8ef9172STejun Heo */
297b8ef9172STejun Heo if (end < dma32_end && dma32_end - end -
298474b881bSTejun Heo mem_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE)
299b8ef9172STejun Heo end = dma32_end;
300b8ef9172STejun Heo
301b8ef9172STejun Heo /*
302b8ef9172STejun Heo * If there won't be enough non-reserved memory for the
303b8ef9172STejun Heo * next node, this one must extend to the end of the
304b8ef9172STejun Heo * physical node.
305b8ef9172STejun Heo */
306cc9aec03SDan Williams if ((limit - end - mem_hole_size(end, limit) < size)
307cc9aec03SDan Williams && !uniform)
308b8ef9172STejun Heo end = limit;
309b8ef9172STejun Heo
310b8ef9172STejun Heo ret = emu_setup_memblk(ei, pi, nid++ % MAX_NUMNODES,
311b8ef9172STejun Heo phys_blk,
312b8ef9172STejun Heo min(end, limit) - start);
313b8ef9172STejun Heo if (ret < 0)
314b8ef9172STejun Heo return ret;
315b8ef9172STejun Heo }
316b8ef9172STejun Heo }
317cc9aec03SDan Williams return nid;
318cc9aec03SDan Williams }
319cc9aec03SDan Williams
split_nodes_size_interleave(struct numa_meminfo * ei,struct numa_meminfo * pi,u64 addr,u64 max_addr,u64 size)320cc9aec03SDan Williams static int __init split_nodes_size_interleave(struct numa_meminfo *ei,
321cc9aec03SDan Williams struct numa_meminfo *pi,
322cc9aec03SDan Williams u64 addr, u64 max_addr, u64 size)
323cc9aec03SDan Williams {
324cc9aec03SDan Williams return split_nodes_size_interleave_uniform(ei, pi, addr, max_addr, size,
325ccae0f36SHuang Ying 0, NULL, 0);
326b8ef9172STejun Heo }
327b8ef9172STejun Heo
setup_emu2phys_nid(int * dfl_phys_nid)328be98dc6eSBenjamin Thiel static int __init setup_emu2phys_nid(int *dfl_phys_nid)
329158f424fSWei Yang {
330158f424fSWei Yang int i, max_emu_nid = 0;
331158f424fSWei Yang
332158f424fSWei Yang *dfl_phys_nid = NUMA_NO_NODE;
333158f424fSWei Yang for (i = 0; i < ARRAY_SIZE(emu_nid_to_phys); i++) {
334158f424fSWei Yang if (emu_nid_to_phys[i] != NUMA_NO_NODE) {
335158f424fSWei Yang max_emu_nid = i;
336158f424fSWei Yang if (*dfl_phys_nid == NUMA_NO_NODE)
337158f424fSWei Yang *dfl_phys_nid = emu_nid_to_phys[i];
338158f424fSWei Yang }
339158f424fSWei Yang }
340158f424fSWei Yang
341158f424fSWei Yang return max_emu_nid;
342158f424fSWei Yang }
343158f424fSWei Yang
34490e6b677STejun Heo /**
34590e6b677STejun Heo * numa_emulation - Emulate NUMA nodes
34690e6b677STejun Heo * @numa_meminfo: NUMA configuration to massage
34790e6b677STejun Heo * @numa_dist_cnt: The size of the physical NUMA distance table
34890e6b677STejun Heo *
34990e6b677STejun Heo * Emulate NUMA nodes according to the numa=fake kernel parameter.
35090e6b677STejun Heo * @numa_meminfo contains the physical memory configuration and is modified
35190e6b677STejun Heo * to reflect the emulated configuration on success. @numa_dist_cnt is
35290e6b677STejun Heo * used to determine the size of the physical distance table.
35390e6b677STejun Heo *
35490e6b677STejun Heo * On success, the following modifications are made.
35590e6b677STejun Heo *
35690e6b677STejun Heo * - @numa_meminfo is updated to reflect the emulated nodes.
35790e6b677STejun Heo *
35890e6b677STejun Heo * - __apicid_to_node[] is updated such that APIC IDs are mapped to the
35990e6b677STejun Heo * emulated nodes.
36090e6b677STejun Heo *
36190e6b677STejun Heo * - NUMA distance table is rebuilt to represent distances between emulated
36290e6b677STejun Heo * nodes. The distances are determined considering how emulated nodes
36390e6b677STejun Heo * are mapped to physical nodes and match the actual distances.
36490e6b677STejun Heo *
36590e6b677STejun Heo * - emu_nid_to_phys[] reflects how emulated nodes are mapped to physical
36690e6b677STejun Heo * nodes. This is used by numa_add_cpu() and numa_remove_cpu().
36790e6b677STejun Heo *
36890e6b677STejun Heo * If emulation is not enabled or fails, emu_nid_to_phys[] is filled with
36990e6b677STejun Heo * identity mapping and no other modification is made.
370b8ef9172STejun Heo */
numa_emulation(struct numa_meminfo * numa_meminfo,int numa_dist_cnt)371b8ef9172STejun Heo void __init numa_emulation(struct numa_meminfo *numa_meminfo, int numa_dist_cnt)
372b8ef9172STejun Heo {
373b8ef9172STejun Heo static struct numa_meminfo ei __initdata;
374b8ef9172STejun Heo static struct numa_meminfo pi __initdata;
3751b7e03efSTejun Heo const u64 max_addr = PFN_PHYS(max_pfn);
376b8ef9172STejun Heo u8 *phys_dist = NULL;
377ce003330SYinghai Lu size_t phys_size = numa_dist_cnt * numa_dist_cnt * sizeof(phys_dist[0]);
37856396e68STejun Heo int max_emu_nid, dfl_phys_nid;
379b8ef9172STejun Heo int i, j, ret;
380b8ef9172STejun Heo
381b8ef9172STejun Heo if (!emu_cmdline)
382b8ef9172STejun Heo goto no_emu;
383b8ef9172STejun Heo
384b8ef9172STejun Heo memset(&ei, 0, sizeof(ei));
385b8ef9172STejun Heo pi = *numa_meminfo;
386b8ef9172STejun Heo
387b8ef9172STejun Heo for (i = 0; i < MAX_NUMNODES; i++)
388b8ef9172STejun Heo emu_nid_to_phys[i] = NUMA_NO_NODE;
389b8ef9172STejun Heo
390b8ef9172STejun Heo /*
391b8ef9172STejun Heo * If the numa=fake command-line contains a 'M' or 'G', it represents
392b8ef9172STejun Heo * the fixed node size. Otherwise, if it is just a single number N,
393b8ef9172STejun Heo * split the system RAM into N fake nodes.
394b8ef9172STejun Heo */
395cc9aec03SDan Williams if (strchr(emu_cmdline, 'U')) {
396cc9aec03SDan Williams nodemask_t physnode_mask = numa_nodes_parsed;
397cc9aec03SDan Williams unsigned long n;
398cc9aec03SDan Williams int nid = 0;
399cc9aec03SDan Williams
400cc9aec03SDan Williams n = simple_strtoul(emu_cmdline, &emu_cmdline, 0);
401cc9aec03SDan Williams ret = -1;
402cc9aec03SDan Williams for_each_node_mask(i, physnode_mask) {
403c6ee7a54SDave Jiang /*
404c6ee7a54SDave Jiang * The reason we pass in blk[0] is due to
405c6ee7a54SDave Jiang * numa_remove_memblk_from() called by
406c6ee7a54SDave Jiang * emu_setup_memblk() will delete entry 0
407c6ee7a54SDave Jiang * and then move everything else up in the pi.blk
408c6ee7a54SDave Jiang * array. Therefore we should always be looking
409c6ee7a54SDave Jiang * at blk[0].
410c6ee7a54SDave Jiang */
411cc9aec03SDan Williams ret = split_nodes_size_interleave_uniform(&ei, &pi,
412c6ee7a54SDave Jiang pi.blk[0].start, pi.blk[0].end, 0,
413c6ee7a54SDave Jiang n, &pi.blk[0], nid);
414cc9aec03SDan Williams if (ret < 0)
415cc9aec03SDan Williams break;
416cc9aec03SDan Williams if (ret < n) {
417cc9aec03SDan Williams pr_info("%s: phys: %d only got %d of %ld nodes, failing\n",
418cc9aec03SDan Williams __func__, i, ret, n);
419cc9aec03SDan Williams ret = -1;
420cc9aec03SDan Williams break;
421cc9aec03SDan Williams }
422cc9aec03SDan Williams nid = ret;
423cc9aec03SDan Williams }
424cc9aec03SDan Williams } else if (strchr(emu_cmdline, 'M') || strchr(emu_cmdline, 'G')) {
425b8ef9172STejun Heo u64 size;
426b8ef9172STejun Heo
427b8ef9172STejun Heo size = memparse(emu_cmdline, &emu_cmdline);
428b8ef9172STejun Heo ret = split_nodes_size_interleave(&ei, &pi, 0, max_addr, size);
429b8ef9172STejun Heo } else {
430b8ef9172STejun Heo unsigned long n;
431b8ef9172STejun Heo
43294c0dd32SPeter Zijlstra n = simple_strtoul(emu_cmdline, &emu_cmdline, 0);
433b8ef9172STejun Heo ret = split_nodes_interleave(&ei, &pi, 0, max_addr, n);
434b8ef9172STejun Heo }
43594c0dd32SPeter Zijlstra if (*emu_cmdline == ':')
43694c0dd32SPeter Zijlstra emu_cmdline++;
437b8ef9172STejun Heo
438b8ef9172STejun Heo if (ret < 0)
439b8ef9172STejun Heo goto no_emu;
440b8ef9172STejun Heo
441b8ef9172STejun Heo if (numa_cleanup_meminfo(&ei) < 0) {
4428d3bcc44SKefeng Wang pr_warn("NUMA: Warning: constructed meminfo invalid, disabling emulation\n");
443b8ef9172STejun Heo goto no_emu;
444b8ef9172STejun Heo }
445b8ef9172STejun Heo
446ce003330SYinghai Lu /* copy the physical distance table */
447b8ef9172STejun Heo if (numa_dist_cnt) {
448b8ef9172STejun Heo u64 phys;
449b8ef9172STejun Heo
450a7259df7SMike Rapoport phys = memblock_phys_alloc_range(phys_size, PAGE_SIZE, 0,
451a7259df7SMike Rapoport PFN_PHYS(max_pfn_mapped));
4521f5026a7STejun Heo if (!phys) {
4538d3bcc44SKefeng Wang pr_warn("NUMA: Warning: can't allocate copy of distance table, disabling emulation\n");
454b8ef9172STejun Heo goto no_emu;
455b8ef9172STejun Heo }
456b8ef9172STejun Heo phys_dist = __va(phys);
457b8ef9172STejun Heo
458b8ef9172STejun Heo for (i = 0; i < numa_dist_cnt; i++)
459b8ef9172STejun Heo for (j = 0; j < numa_dist_cnt; j++)
460b8ef9172STejun Heo phys_dist[i * numa_dist_cnt + j] =
461b8ef9172STejun Heo node_distance(i, j);
462b8ef9172STejun Heo }
463b8ef9172STejun Heo
46456396e68STejun Heo /*
46556396e68STejun Heo * Determine the max emulated nid and the default phys nid to use
46656396e68STejun Heo * for unmapped nodes.
46756396e68STejun Heo */
468158f424fSWei Yang max_emu_nid = setup_emu2phys_nid(&dfl_phys_nid);
469078a1989STejun Heo
470b8ef9172STejun Heo /* commit */
471b8ef9172STejun Heo *numa_meminfo = ei;
472b8ef9172STejun Heo
4734f167201SWei Yang /* Make sure numa_nodes_parsed only contains emulated nodes */
4744f167201SWei Yang nodes_clear(numa_nodes_parsed);
4754f167201SWei Yang for (i = 0; i < ARRAY_SIZE(ei.blk); i++)
4764f167201SWei Yang if (ei.blk[i].start != ei.blk[i].end &&
4774f167201SWei Yang ei.blk[i].nid != NUMA_NO_NODE)
4784f167201SWei Yang node_set(ei.blk[i].nid, numa_nodes_parsed);
4794f167201SWei Yang
480b8ef9172STejun Heo /*
481b8ef9172STejun Heo * Transform __apicid_to_node table to use emulated nids by
482b8ef9172STejun Heo * reverse-mapping phys_nid. The maps should always exist but fall
483b8ef9172STejun Heo * back to zero just in case.
484b8ef9172STejun Heo */
485b8ef9172STejun Heo for (i = 0; i < ARRAY_SIZE(__apicid_to_node); i++) {
486b8ef9172STejun Heo if (__apicid_to_node[i] == NUMA_NO_NODE)
487b8ef9172STejun Heo continue;
488b8ef9172STejun Heo for (j = 0; j < ARRAY_SIZE(emu_nid_to_phys); j++)
489b8ef9172STejun Heo if (__apicid_to_node[i] == emu_nid_to_phys[j])
490b8ef9172STejun Heo break;
491b8ef9172STejun Heo __apicid_to_node[i] = j < ARRAY_SIZE(emu_nid_to_phys) ? j : 0;
492b8ef9172STejun Heo }
493b8ef9172STejun Heo
494b8ef9172STejun Heo /* make sure all emulated nodes are mapped to a physical node */
495b8ef9172STejun Heo for (i = 0; i < ARRAY_SIZE(emu_nid_to_phys); i++)
496b8ef9172STejun Heo if (emu_nid_to_phys[i] == NUMA_NO_NODE)
497078a1989STejun Heo emu_nid_to_phys[i] = dfl_phys_nid;
498b8ef9172STejun Heo
49956396e68STejun Heo /* transform distance table */
500b8ef9172STejun Heo numa_reset_distance();
50156396e68STejun Heo for (i = 0; i < max_emu_nid + 1; i++) {
50256396e68STejun Heo for (j = 0; j < max_emu_nid + 1; j++) {
503b8ef9172STejun Heo int physi = emu_nid_to_phys[i];
504b8ef9172STejun Heo int physj = emu_nid_to_phys[j];
505b8ef9172STejun Heo int dist;
506b8ef9172STejun Heo
50794c0dd32SPeter Zijlstra if (get_option(&emu_cmdline, &dist) == 2)
50894c0dd32SPeter Zijlstra ;
50994c0dd32SPeter Zijlstra else if (physi >= numa_dist_cnt || physj >= numa_dist_cnt)
510b8ef9172STejun Heo dist = physi == physj ?
511b8ef9172STejun Heo LOCAL_DISTANCE : REMOTE_DISTANCE;
512b8ef9172STejun Heo else
513b8ef9172STejun Heo dist = phys_dist[physi * numa_dist_cnt + physj];
514b8ef9172STejun Heo
515b8ef9172STejun Heo numa_set_distance(i, j, dist);
516b8ef9172STejun Heo }
517b8ef9172STejun Heo }
518ce003330SYinghai Lu
519ce003330SYinghai Lu /* free the copied physical distance table */
5204421cca0SMike Rapoport memblock_free(phys_dist, phys_size);
521b8ef9172STejun Heo return;
522b8ef9172STejun Heo
523b8ef9172STejun Heo no_emu:
524b8ef9172STejun Heo /* No emulation. Build identity emu_nid_to_phys[] for numa_add_cpu() */
525b8ef9172STejun Heo for (i = 0; i < ARRAY_SIZE(emu_nid_to_phys); i++)
526b8ef9172STejun Heo emu_nid_to_phys[i] = i;
527b8ef9172STejun Heo }
528b8ef9172STejun Heo
529b8ef9172STejun Heo #ifndef CONFIG_DEBUG_PER_CPU_MAPS
numa_add_cpu(int cpu)530148f9bb8SPaul Gortmaker void numa_add_cpu(int cpu)
531b8ef9172STejun Heo {
532b8ef9172STejun Heo int physnid, nid;
533b8ef9172STejun Heo
534b8ef9172STejun Heo nid = early_cpu_to_node(cpu);
535b8ef9172STejun Heo BUG_ON(nid == NUMA_NO_NODE || !node_online(nid));
536b8ef9172STejun Heo
537b8ef9172STejun Heo physnid = emu_nid_to_phys[nid];
538b8ef9172STejun Heo
539b8ef9172STejun Heo /*
540b8ef9172STejun Heo * Map the cpu to each emulated node that is allocated on the physical
541b8ef9172STejun Heo * node of the cpu's apic id.
542b8ef9172STejun Heo */
543b8ef9172STejun Heo for_each_online_node(nid)
544b8ef9172STejun Heo if (emu_nid_to_phys[nid] == physnid)
545b8ef9172STejun Heo cpumask_set_cpu(cpu, node_to_cpumask_map[nid]);
546b8ef9172STejun Heo }
547b8ef9172STejun Heo
numa_remove_cpu(int cpu)548148f9bb8SPaul Gortmaker void numa_remove_cpu(int cpu)
549b8ef9172STejun Heo {
550b8ef9172STejun Heo int i;
551b8ef9172STejun Heo
552b8ef9172STejun Heo for_each_online_node(i)
553b8ef9172STejun Heo cpumask_clear_cpu(cpu, node_to_cpumask_map[i]);
554b8ef9172STejun Heo }
555b8ef9172STejun Heo #else /* !CONFIG_DEBUG_PER_CPU_MAPS */
numa_set_cpumask(int cpu,bool enable)556148f9bb8SPaul Gortmaker static void numa_set_cpumask(int cpu, bool enable)
557b8ef9172STejun Heo {
5587a6c6547SDavid Rientjes int nid, physnid;
559b8ef9172STejun Heo
560b8ef9172STejun Heo nid = early_cpu_to_node(cpu);
561b8ef9172STejun Heo if (nid == NUMA_NO_NODE) {
562b8ef9172STejun Heo /* early_cpu_to_node() already emits a warning and trace */
563b8ef9172STejun Heo return;
564b8ef9172STejun Heo }
565b8ef9172STejun Heo
566b8ef9172STejun Heo physnid = emu_nid_to_phys[nid];
567b8ef9172STejun Heo
5687a6c6547SDavid Rientjes for_each_online_node(nid) {
569b8ef9172STejun Heo if (emu_nid_to_phys[nid] != physnid)
570b8ef9172STejun Heo continue;
571b8ef9172STejun Heo
5727a6c6547SDavid Rientjes debug_cpumask_set_cpu(cpu, nid, enable);
573b8ef9172STejun Heo }
574b8ef9172STejun Heo }
575b8ef9172STejun Heo
numa_add_cpu(int cpu)576148f9bb8SPaul Gortmaker void numa_add_cpu(int cpu)
577b8ef9172STejun Heo {
5787a6c6547SDavid Rientjes numa_set_cpumask(cpu, true);
579b8ef9172STejun Heo }
580b8ef9172STejun Heo
numa_remove_cpu(int cpu)581148f9bb8SPaul Gortmaker void numa_remove_cpu(int cpu)
582b8ef9172STejun Heo {
5837a6c6547SDavid Rientjes numa_set_cpumask(cpu, false);
584b8ef9172STejun Heo }
585b8ef9172STejun Heo #endif /* !CONFIG_DEBUG_PER_CPU_MAPS */
586