1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved.
4  * Author: Marc Zyngier <marc.zyngier@arm.com>
5  */
6 
7 #include <linux/interrupt.h>
8 #include <linux/irq.h>
9 #include <linux/irqdomain.h>
10 #include <linux/msi.h>
11 #include <linux/sched.h>
12 
13 #include <linux/irqchip/arm-gic-v4.h>
14 
15 /*
16  * WARNING: The blurb below assumes that you understand the
17  * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets
18  * translated into GICv4 commands. So it effectively targets at most
19  * two individuals. You know who you are.
20  *
21  * The core GICv4 code is designed to *avoid* exposing too much of the
22  * core GIC code (that would in turn leak into the hypervisor code),
23  * and instead provide a hypervisor agnostic interface to the HW (of
24  * course, the astute reader will quickly realize that hypervisor
25  * agnostic actually means KVM-specific - what were you thinking?).
26  *
27  * In order to achieve a modicum of isolation, we try to hide most of
28  * the GICv4 "stuff" behind normal irqchip operations:
29  *
30  * - Any guest-visible VLPI is backed by a Linux interrupt (and a
31  *   physical LPI which gets unmapped when the guest maps the
32  *   VLPI). This allows the same DevID/EventID pair to be either
33  *   mapped to the LPI (host) or the VLPI (guest). Note that this is
34  *   exclusive, and you cannot have both.
35  *
36  * - Enabling/disabling a VLPI is done by issuing mask/unmask calls.
37  *
38  * - Guest INT/CLEAR commands are implemented through
39  *   irq_set_irqchip_state().
40  *
41  * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or
42  *   issuing an INV after changing a priority) gets shoved into the
43  *   irq_set_vcpu_affinity() method. While this is quite horrible
44  *   (let's face it, this is the irqchip version of an ioctl), it
45  *   confines the crap to a single location. And map/unmap really is
46  *   about setting the affinity of a VLPI to a vcpu, so only INV is
47  *   majorly out of place. So there.
48  *
49  * A number of commands are simply not provided by this interface, as
50  * they do not make direct sense. For example, MAPD is purely local to
51  * the virtual ITS (because it references a virtual device, and the
52  * physical ITS is still very much in charge of the physical
53  * device). Same goes for things like MAPC (the physical ITS deals
54  * with the actual vPE affinity, and not the braindead concept of
55  * collection). SYNC is not provided either, as each and every command
56  * is followed by a VSYNC. This could be relaxed in the future, should
57  * this be seen as a bottleneck (yes, this means *never*).
58  *
59  * But handling VLPIs is only one side of the job of the GICv4
60  * code. The other (darker) side is to take care of the doorbell
61  * interrupts which are delivered when a VLPI targeting a non-running
62  * vcpu is being made pending.
63  *
64  * The choice made here is that each vcpu (VPE in old northern GICv4
65  * dialect) gets a single doorbell LPI, no matter how many interrupts
66  * are targeting it. This has a nice property, which is that the
67  * interrupt becomes a handle for the VPE, and that the hypervisor
68  * code can manipulate it through the normal interrupt API:
69  *
70  * - VMs (or rather the VM abstraction that matters to the GIC)
71  *   contain an irq domain where each interrupt maps to a VPE. In
72  *   turn, this domain sits on top of the normal LPI allocator, and a
73  *   specially crafted irq_chip implementation.
74  *
75  * - mask/unmask do what is expected on the doorbell interrupt.
76  *
77  * - irq_set_affinity is used to move a VPE from one redistributor to
78  *   another.
79  *
80  * - irq_set_vcpu_affinity once again gets hijacked for the purpose of
81  *   creating a new sub-API, namely scheduling/descheduling a VPE
82  *   (which involves programming GICR_V{PROP,PEND}BASER) and
83  *   performing INVALL operations.
84  */
85 
86 static struct irq_domain *gic_domain;
87 static const struct irq_domain_ops *vpe_domain_ops;
88 static const struct irq_domain_ops *sgi_domain_ops;
89 
90 #ifdef CONFIG_ARM64
91 #include <asm/cpufeature.h>
92 
gic_cpuif_has_vsgi(void)93 bool gic_cpuif_has_vsgi(void)
94 {
95 	unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
96 
97 	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_GIC_SHIFT);
98 
99 	return fld >= 0x3;
100 }
101 #else
gic_cpuif_has_vsgi(void)102 bool gic_cpuif_has_vsgi(void)
103 {
104 	return false;
105 }
106 #endif
107 
has_v4_1(void)108 static bool has_v4_1(void)
109 {
110 	return !!sgi_domain_ops;
111 }
112 
has_v4_1_sgi(void)113 static bool has_v4_1_sgi(void)
114 {
115 	return has_v4_1() && gic_cpuif_has_vsgi();
116 }
117 
its_alloc_vcpu_sgis(struct its_vpe * vpe,int idx)118 static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx)
119 {
120 	char *name;
121 	int sgi_base;
122 
123 	if (!has_v4_1_sgi())
124 		return 0;
125 
126 	name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current));
127 	if (!name)
128 		goto err;
129 
130 	vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx);
131 	if (!vpe->fwnode)
132 		goto err;
133 
134 	kfree(name);
135 	name = NULL;
136 
137 	vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16,
138 						   sgi_domain_ops, vpe);
139 	if (!vpe->sgi_domain)
140 		goto err;
141 
142 	sgi_base = __irq_domain_alloc_irqs(vpe->sgi_domain, -1, 16,
143 					       NUMA_NO_NODE, vpe,
144 					       false, NULL);
145 	if (sgi_base <= 0)
146 		goto err;
147 
148 	return 0;
149 
150 err:
151 	if (vpe->sgi_domain)
152 		irq_domain_remove(vpe->sgi_domain);
153 	if (vpe->fwnode)
154 		irq_domain_free_fwnode(vpe->fwnode);
155 	kfree(name);
156 	return -ENOMEM;
157 }
158 
its_alloc_vcpu_irqs(struct its_vm * vm)159 int its_alloc_vcpu_irqs(struct its_vm *vm)
160 {
161 	int vpe_base_irq, i;
162 
163 	vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe",
164 						      task_pid_nr(current));
165 	if (!vm->fwnode)
166 		goto err;
167 
168 	vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes,
169 						 vm->fwnode, vpe_domain_ops,
170 						 vm);
171 	if (!vm->domain)
172 		goto err;
173 
174 	for (i = 0; i < vm->nr_vpes; i++) {
175 		vm->vpes[i]->its_vm = vm;
176 		vm->vpes[i]->idai = true;
177 	}
178 
179 	vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes,
180 					       NUMA_NO_NODE, vm,
181 					       false, NULL);
182 	if (vpe_base_irq <= 0)
183 		goto err;
184 
185 	for (i = 0; i < vm->nr_vpes; i++) {
186 		int ret;
187 		vm->vpes[i]->irq = vpe_base_irq + i;
188 		ret = its_alloc_vcpu_sgis(vm->vpes[i], i);
189 		if (ret)
190 			goto err;
191 	}
192 
193 	return 0;
194 
195 err:
196 	if (vm->domain)
197 		irq_domain_remove(vm->domain);
198 	if (vm->fwnode)
199 		irq_domain_free_fwnode(vm->fwnode);
200 
201 	return -ENOMEM;
202 }
203 
its_free_sgi_irqs(struct its_vm * vm)204 static void its_free_sgi_irqs(struct its_vm *vm)
205 {
206 	int i;
207 
208 	if (!has_v4_1_sgi())
209 		return;
210 
211 	for (i = 0; i < vm->nr_vpes; i++) {
212 		unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0);
213 
214 		if (WARN_ON(!irq))
215 			continue;
216 
217 		irq_domain_free_irqs(irq, 16);
218 		irq_domain_remove(vm->vpes[i]->sgi_domain);
219 		irq_domain_free_fwnode(vm->vpes[i]->fwnode);
220 	}
221 }
222 
its_free_vcpu_irqs(struct its_vm * vm)223 void its_free_vcpu_irqs(struct its_vm *vm)
224 {
225 	its_free_sgi_irqs(vm);
226 	irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes);
227 	irq_domain_remove(vm->domain);
228 	irq_domain_free_fwnode(vm->fwnode);
229 }
230 
its_send_vpe_cmd(struct its_vpe * vpe,struct its_cmd_info * info)231 static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
232 {
233 	return irq_set_vcpu_affinity(vpe->irq, info);
234 }
235 
its_make_vpe_non_resident(struct its_vpe * vpe,bool db)236 int its_make_vpe_non_resident(struct its_vpe *vpe, bool db)
237 {
238 	struct irq_desc *desc = irq_to_desc(vpe->irq);
239 	struct its_cmd_info info = { };
240 	int ret;
241 
242 	WARN_ON(preemptible());
243 
244 	info.cmd_type = DESCHEDULE_VPE;
245 	if (has_v4_1()) {
246 		/* GICv4.1 can directly deal with doorbells */
247 		info.req_db = db;
248 	} else {
249 		/* Undo the nested disable_irq() calls... */
250 		while (db && irqd_irq_disabled(&desc->irq_data))
251 			enable_irq(vpe->irq);
252 	}
253 
254 	ret = its_send_vpe_cmd(vpe, &info);
255 	if (!ret)
256 		vpe->resident = false;
257 
258 	vpe->ready = false;
259 
260 	return ret;
261 }
262 
its_make_vpe_resident(struct its_vpe * vpe,bool g0en,bool g1en)263 int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en)
264 {
265 	struct its_cmd_info info = { };
266 	int ret;
267 
268 	WARN_ON(preemptible());
269 
270 	info.cmd_type = SCHEDULE_VPE;
271 	if (has_v4_1()) {
272 		info.g0en = g0en;
273 		info.g1en = g1en;
274 	} else {
275 		/* Disabled the doorbell, as we're about to enter the guest */
276 		disable_irq_nosync(vpe->irq);
277 	}
278 
279 	ret = its_send_vpe_cmd(vpe, &info);
280 	if (!ret)
281 		vpe->resident = true;
282 
283 	return ret;
284 }
285 
its_commit_vpe(struct its_vpe * vpe)286 int its_commit_vpe(struct its_vpe *vpe)
287 {
288 	struct its_cmd_info info = {
289 		.cmd_type = COMMIT_VPE,
290 	};
291 	int ret;
292 
293 	WARN_ON(preemptible());
294 
295 	ret = its_send_vpe_cmd(vpe, &info);
296 	if (!ret)
297 		vpe->ready = true;
298 
299 	return ret;
300 }
301 
302 
its_invall_vpe(struct its_vpe * vpe)303 int its_invall_vpe(struct its_vpe *vpe)
304 {
305 	struct its_cmd_info info = {
306 		.cmd_type = INVALL_VPE,
307 	};
308 
309 	return its_send_vpe_cmd(vpe, &info);
310 }
311 
its_map_vlpi(int irq,struct its_vlpi_map * map)312 int its_map_vlpi(int irq, struct its_vlpi_map *map)
313 {
314 	struct its_cmd_info info = {
315 		.cmd_type = MAP_VLPI,
316 		{
317 			.map      = map,
318 		},
319 	};
320 	int ret;
321 
322 	/*
323 	 * The host will never see that interrupt firing again, so it
324 	 * is vital that we don't do any lazy masking.
325 	 */
326 	irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
327 
328 	ret = irq_set_vcpu_affinity(irq, &info);
329 	if (ret)
330 		irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
331 
332 	return ret;
333 }
334 
its_get_vlpi(int irq,struct its_vlpi_map * map)335 int its_get_vlpi(int irq, struct its_vlpi_map *map)
336 {
337 	struct its_cmd_info info = {
338 		.cmd_type = GET_VLPI,
339 		{
340 			.map      = map,
341 		},
342 	};
343 
344 	return irq_set_vcpu_affinity(irq, &info);
345 }
346 
its_unmap_vlpi(int irq)347 int its_unmap_vlpi(int irq)
348 {
349 	irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
350 	return irq_set_vcpu_affinity(irq, NULL);
351 }
352 
its_prop_update_vlpi(int irq,u8 config,bool inv)353 int its_prop_update_vlpi(int irq, u8 config, bool inv)
354 {
355 	struct its_cmd_info info = {
356 		.cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI,
357 		{
358 			.config   = config,
359 		},
360 	};
361 
362 	return irq_set_vcpu_affinity(irq, &info);
363 }
364 
its_prop_update_vsgi(int irq,u8 priority,bool group)365 int its_prop_update_vsgi(int irq, u8 priority, bool group)
366 {
367 	struct its_cmd_info info = {
368 		.cmd_type = PROP_UPDATE_VSGI,
369 		{
370 			.priority	= priority,
371 			.group		= group,
372 		},
373 	};
374 
375 	return irq_set_vcpu_affinity(irq, &info);
376 }
377 
its_init_v4(struct irq_domain * domain,const struct irq_domain_ops * vpe_ops,const struct irq_domain_ops * sgi_ops)378 int its_init_v4(struct irq_domain *domain,
379 		const struct irq_domain_ops *vpe_ops,
380 		const struct irq_domain_ops *sgi_ops)
381 {
382 	if (domain) {
383 		pr_info("ITS: Enabling GICv4 support\n");
384 		gic_domain = domain;
385 		vpe_domain_ops = vpe_ops;
386 		sgi_domain_ops = sgi_ops;
387 		return 0;
388 	}
389 
390 	pr_err("ITS: No GICv4 VPE domain allocated\n");
391 	return -ENODEV;
392 }
393