1 //! Legalize ABI boundaries.
2 //!
3 //! This legalizer sub-module contains code for dealing with ABI boundaries:
4 //!
5 //! - Function arguments passed to the entry block.
6 //! - Function arguments passed to call instructions.
7 //! - Return values from call instructions.
8 //! - Return values passed to return instructions.
9 //!
10 //! The ABI boundary legalization happens in two phases:
11 //!
12 //! 1. The `legalize_signatures` function rewrites all the preamble signatures with ABI information
13 //! and possibly new argument types. It also rewrites the entry block arguments to match.
14 //! 2. The `handle_call_abi` and `handle_return_abi` functions rewrite call and return instructions
15 //! to match the new ABI signatures.
16 //!
17 //! Between the two phases, preamble signatures and call/return arguments don't match. This
18 //! intermediate state doesn't type check.
19
20 use crate::abi::{legalize_abi_value, ValueConversion};
21 use crate::cursor::{Cursor, FuncCursor};
22 use crate::flowgraph::ControlFlowGraph;
23 use crate::ir::instructions::CallInfo;
24 use crate::ir::{
25 AbiParam, ArgumentLoc, ArgumentPurpose, Block, DataFlowGraph, ExtFuncData, ExternalName,
26 Function, Inst, InstBuilder, LibCall, MemFlags, SigRef, Signature, StackSlotData,
27 StackSlotKind, Type, Value, ValueLoc,
28 };
29 use crate::isa::TargetIsa;
30 use crate::legalizer::split::{isplit, vsplit};
31 use alloc::borrow::Cow;
32 use alloc::vec::Vec;
33 use core::mem;
34 use cranelift_entity::EntityList;
35 use log::debug;
36
37 /// Legalize all the function signatures in `func`.
38 ///
39 /// This changes all signatures to be ABI-compliant with full `ArgumentLoc` annotations. It doesn't
40 /// change the entry block arguments, calls, or return instructions, so this can leave the function
41 /// in a state with type discrepancies.
legalize_signatures(func: &mut Function, isa: &dyn TargetIsa)42 pub fn legalize_signatures(func: &mut Function, isa: &dyn TargetIsa) {
43 if let Some(new) = legalize_signature(&func.signature, true, isa) {
44 let old = mem::replace(&mut func.signature, new);
45 func.old_signature = Some(old);
46 }
47
48 for (sig_ref, sig_data) in func.dfg.signatures.iter_mut() {
49 if let Some(new) = legalize_signature(sig_data, false, isa) {
50 let old = mem::replace(sig_data, new);
51 func.dfg.old_signatures[sig_ref] = Some(old);
52 }
53 }
54
55 if let Some(entry) = func.layout.entry_block() {
56 legalize_entry_params(func, entry);
57 spill_entry_params(func, entry);
58 }
59 }
60
61 /// Legalize the libcall signature, which we may generate on the fly after
62 /// `legalize_signatures` has been called.
legalize_libcall_signature(signature: &mut Signature, isa: &dyn TargetIsa)63 pub fn legalize_libcall_signature(signature: &mut Signature, isa: &dyn TargetIsa) {
64 if let Some(s) = legalize_signature(signature, false, isa) {
65 *signature = s;
66 }
67 }
68
69 /// Legalize the given signature.
70 ///
71 /// `current` is true if this is the signature for the current function.
legalize_signature( signature: &Signature, current: bool, isa: &dyn TargetIsa, ) -> Option<Signature>72 fn legalize_signature(
73 signature: &Signature,
74 current: bool,
75 isa: &dyn TargetIsa,
76 ) -> Option<Signature> {
77 let mut cow = Cow::Borrowed(signature);
78 isa.legalize_signature(&mut cow, current);
79 match cow {
80 Cow::Borrowed(_) => None,
81 Cow::Owned(s) => Some(s),
82 }
83 }
84
85 /// Legalize the entry block parameters after `func`'s signature has been legalized.
86 ///
87 /// The legalized signature may contain more parameters than the original signature, and the
88 /// parameter types have been changed. This function goes through the parameters of the entry block
89 /// and replaces them with parameters of the right type for the ABI.
90 ///
91 /// The original entry block parameters are computed from the new ABI parameters by code inserted at
92 /// the top of the entry block.
legalize_entry_params(func: &mut Function, entry: Block)93 fn legalize_entry_params(func: &mut Function, entry: Block) {
94 let mut has_sret = false;
95 let mut has_link = false;
96 let mut has_vmctx = false;
97 let mut has_sigid = false;
98 let mut has_stack_limit = false;
99
100 // Insert position for argument conversion code.
101 // We want to insert instructions before the first instruction in the entry block.
102 // If the entry block is empty, append instructions to it instead.
103 let mut pos = FuncCursor::new(func).at_first_inst(entry);
104
105 // Keep track of the argument types in the ABI-legalized signature.
106 let mut abi_arg = 0;
107
108 // Process the block parameters one at a time, possibly replacing one argument with multiple new
109 // ones. We do this by detaching the entry block parameters first.
110 let block_params = pos.func.dfg.detach_block_params(entry);
111 let mut old_arg = 0;
112 while let Some(arg) = block_params.get(old_arg, &pos.func.dfg.value_lists) {
113 old_arg += 1;
114
115 let abi_type = pos.func.signature.params[abi_arg];
116 let arg_type = pos.func.dfg.value_type(arg);
117 if let ArgumentPurpose::StructArgument(size) = abi_type.purpose {
118 let offset = if let ArgumentLoc::Stack(offset) = abi_type.location {
119 offset
120 } else {
121 unreachable!("StructArgument must already have a Stack ArgumentLoc assigned");
122 };
123 let ss = pos.func.stack_slots.make_incoming_arg(size, offset);
124 let struct_arg = pos.ins().stack_addr(arg_type, ss, 0);
125 pos.func.dfg.change_to_alias(arg, struct_arg);
126 let dummy = pos
127 .func
128 .dfg
129 .append_block_param(entry, crate::ir::types::SARG_T);
130 pos.func.locations[dummy] = ValueLoc::Stack(ss);
131 abi_arg += 1;
132 continue;
133 }
134
135 if arg_type == abi_type.value_type {
136 // No value translation is necessary, this argument matches the ABI type.
137 // Just use the original block argument value. This is the most common case.
138 pos.func.dfg.attach_block_param(entry, arg);
139 match abi_type.purpose {
140 ArgumentPurpose::Normal => {}
141 ArgumentPurpose::StructArgument(_) => unreachable!("Handled above"),
142 ArgumentPurpose::FramePointer => {}
143 ArgumentPurpose::CalleeSaved => {}
144 ArgumentPurpose::StructReturn => {
145 debug_assert!(!has_sret, "Multiple sret arguments found");
146 has_sret = true;
147 }
148 ArgumentPurpose::VMContext => {
149 debug_assert!(!has_vmctx, "Multiple vmctx arguments found");
150 has_vmctx = true;
151 }
152 ArgumentPurpose::SignatureId => {
153 debug_assert!(!has_sigid, "Multiple sigid arguments found");
154 has_sigid = true;
155 }
156 ArgumentPurpose::StackLimit => {
157 debug_assert!(!has_stack_limit, "Multiple stack_limit arguments found");
158 has_stack_limit = true;
159 }
160 ArgumentPurpose::Link => panic!("Unexpected link arg {}", abi_type),
161 ArgumentPurpose::CallerTLS | ArgumentPurpose::CalleeTLS => {}
162 }
163 abi_arg += 1;
164 } else {
165 // Compute the value we want for `arg` from the legalized ABI parameters.
166 let mut get_arg = |func: &mut Function, ty| {
167 let abi_type = func.signature.params[abi_arg];
168 debug_assert_eq!(
169 abi_type.purpose,
170 ArgumentPurpose::Normal,
171 "Can't legalize special-purpose argument"
172 );
173 if ty == abi_type.value_type {
174 abi_arg += 1;
175 Ok(func.dfg.append_block_param(entry, ty))
176 } else {
177 Err(abi_type)
178 }
179 };
180 let converted = convert_from_abi(&mut pos, arg_type, Some(arg), &mut get_arg);
181 // The old `arg` is no longer an attached block argument, but there are probably still
182 // uses of the value.
183 debug_assert_eq!(pos.func.dfg.resolve_aliases(arg), converted);
184 }
185 }
186
187 // The legalized signature may contain additional parameters representing special-purpose
188 // registers.
189 for &arg in &pos.func.signature.params[abi_arg..] {
190 match arg.purpose {
191 // Any normal parameters should have been processed above.
192 ArgumentPurpose::Normal | ArgumentPurpose::StructArgument(_) => {
193 panic!("Leftover arg: {}", arg);
194 }
195 // The callee-save parameters should not appear until after register allocation is
196 // done.
197 ArgumentPurpose::FramePointer | ArgumentPurpose::CalleeSaved => {
198 panic!("Premature callee-saved arg {}", arg);
199 }
200 // These can be meaningfully added by `legalize_signature()`.
201 ArgumentPurpose::Link => {
202 debug_assert!(!has_link, "Multiple link parameters found");
203 has_link = true;
204 }
205 ArgumentPurpose::StructReturn => {
206 debug_assert!(!has_sret, "Multiple sret parameters found");
207 has_sret = true;
208 }
209 ArgumentPurpose::VMContext => {
210 debug_assert!(!has_vmctx, "Multiple vmctx parameters found");
211 has_vmctx = true;
212 }
213 ArgumentPurpose::SignatureId => {
214 debug_assert!(!has_sigid, "Multiple sigid parameters found");
215 has_sigid = true;
216 }
217 ArgumentPurpose::StackLimit => {
218 debug_assert!(!has_stack_limit, "Multiple stack_limit parameters found");
219 has_stack_limit = true;
220 }
221 ArgumentPurpose::CallerTLS | ArgumentPurpose::CalleeTLS => {}
222 }
223
224 // Just create entry block values to match here. We will use them in `handle_return_abi()`
225 // below.
226 pos.func.dfg.append_block_param(entry, arg.value_type);
227 }
228 }
229
230 /// Legalize the results returned from a call instruction to match the ABI signature.
231 ///
232 /// The cursor `pos` points to a call instruction with at least one return value. The cursor will
233 /// be left pointing after the instructions inserted to convert the return values.
234 ///
235 /// This function is very similar to the `legalize_entry_params` function above.
236 ///
237 /// Returns the possibly new instruction representing the call.
legalize_inst_results<ResType>(pos: &mut FuncCursor, mut get_abi_type: ResType) -> Inst where ResType: FnMut(&Function, usize) -> AbiParam,238 fn legalize_inst_results<ResType>(pos: &mut FuncCursor, mut get_abi_type: ResType) -> Inst
239 where
240 ResType: FnMut(&Function, usize) -> AbiParam,
241 {
242 let call = pos
243 .current_inst()
244 .expect("Cursor must point to a call instruction");
245
246 // We theoretically allow for call instructions that return a number of fixed results before
247 // the call return values. In practice, it doesn't happen.
248 debug_assert_eq!(
249 pos.func.dfg[call]
250 .opcode()
251 .constraints()
252 .num_fixed_results(),
253 0,
254 "Fixed results on calls not supported"
255 );
256
257 let results = pos.func.dfg.detach_results(call);
258 let mut next_res = 0;
259 let mut abi_res = 0;
260
261 // Point immediately after the call.
262 pos.next_inst();
263
264 while let Some(res) = results.get(next_res, &pos.func.dfg.value_lists) {
265 next_res += 1;
266
267 let res_type = pos.func.dfg.value_type(res);
268 if res_type == get_abi_type(pos.func, abi_res).value_type {
269 // No value translation is necessary, this result matches the ABI type.
270 pos.func.dfg.attach_result(call, res);
271 abi_res += 1;
272 } else {
273 let mut get_res = |func: &mut Function, ty| {
274 let abi_type = get_abi_type(func, abi_res);
275 if ty == abi_type.value_type {
276 let last_res = func.dfg.append_result(call, ty);
277 abi_res += 1;
278 Ok(last_res)
279 } else {
280 Err(abi_type)
281 }
282 };
283 let v = convert_from_abi(pos, res_type, Some(res), &mut get_res);
284 debug_assert_eq!(pos.func.dfg.resolve_aliases(res), v);
285 }
286 }
287
288 call
289 }
290
assert_is_valid_sret_legalization( old_ret_list: &EntityList<Value>, old_sig: &Signature, new_sig: &Signature, pos: &FuncCursor, )291 fn assert_is_valid_sret_legalization(
292 old_ret_list: &EntityList<Value>,
293 old_sig: &Signature,
294 new_sig: &Signature,
295 pos: &FuncCursor,
296 ) {
297 debug_assert_eq!(
298 old_sig.returns.len(),
299 old_ret_list.len(&pos.func.dfg.value_lists)
300 );
301
302 // Assert that the only difference in special parameters is that there
303 // is an appended struct return pointer parameter.
304 let old_special_params: Vec<_> = old_sig
305 .params
306 .iter()
307 .filter(|r| r.purpose != ArgumentPurpose::Normal)
308 .collect();
309 let new_special_params: Vec<_> = new_sig
310 .params
311 .iter()
312 .filter(|r| r.purpose != ArgumentPurpose::Normal)
313 .collect();
314 debug_assert_eq!(old_special_params.len() + 1, new_special_params.len());
315 debug_assert!(old_special_params
316 .iter()
317 .zip(&new_special_params)
318 .all(|(old, new)| old.purpose == new.purpose));
319 debug_assert_eq!(
320 new_special_params.last().unwrap().purpose,
321 ArgumentPurpose::StructReturn
322 );
323
324 // If the special returns have changed at all, then the only change
325 // should be that the struct return pointer is returned back out of the
326 // function, so that callers don't have to load its stack address again.
327 let old_special_returns: Vec<_> = old_sig
328 .returns
329 .iter()
330 .filter(|r| r.purpose != ArgumentPurpose::Normal)
331 .collect();
332 let new_special_returns: Vec<_> = new_sig
333 .returns
334 .iter()
335 .filter(|r| r.purpose != ArgumentPurpose::Normal)
336 .collect();
337 debug_assert!(old_special_returns
338 .iter()
339 .zip(&new_special_returns)
340 .all(|(old, new)| old.purpose == new.purpose));
341 debug_assert!(
342 old_special_returns.len() == new_special_returns.len()
343 || (old_special_returns.len() + 1 == new_special_returns.len()
344 && new_special_returns.last().unwrap().purpose == ArgumentPurpose::StructReturn)
345 );
346 }
347
legalize_sret_call(isa: &dyn TargetIsa, pos: &mut FuncCursor, sig_ref: SigRef, call: Inst)348 fn legalize_sret_call(isa: &dyn TargetIsa, pos: &mut FuncCursor, sig_ref: SigRef, call: Inst) {
349 let old_ret_list = pos.func.dfg.detach_results(call);
350 let old_sig = pos.func.dfg.old_signatures[sig_ref]
351 .take()
352 .expect("must have an old signature when using an `sret` parameter");
353
354 // We make a bunch of assumptions about the shape of the old, multi-return
355 // signature and the new, sret-using signature in this legalization
356 // function. Assert that these assumptions hold true in debug mode.
357 if cfg!(debug_assertions) {
358 assert_is_valid_sret_legalization(
359 &old_ret_list,
360 &old_sig,
361 &pos.func.dfg.signatures[sig_ref],
362 &pos,
363 );
364 }
365
366 // Go through and remove all normal return values from the `call`
367 // instruction's returns list. These will be stored into the stack slot that
368 // the sret points to. At the same time, calculate the size of the sret
369 // stack slot.
370 let mut sret_slot_size = 0;
371 for (i, ret) in old_sig.returns.iter().enumerate() {
372 let v = old_ret_list.get(i, &pos.func.dfg.value_lists).unwrap();
373 let ty = pos.func.dfg.value_type(v);
374 if ret.purpose == ArgumentPurpose::Normal {
375 debug_assert_eq!(ret.location, ArgumentLoc::Unassigned);
376 let ty = legalized_type_for_sret(ty);
377 let size = ty.bytes();
378 sret_slot_size = round_up_to_multiple_of_type_align(sret_slot_size, ty) + size;
379 } else {
380 let new_v = pos.func.dfg.append_result(call, ty);
381 pos.func.dfg.change_to_alias(v, new_v);
382 }
383 }
384
385 let stack_slot = pos.func.stack_slots.push(StackSlotData {
386 kind: StackSlotKind::StructReturnSlot,
387 size: sret_slot_size,
388 offset: None,
389 });
390
391 // Append the sret pointer to the `call` instruction's arguments.
392 let ptr_type = Type::triple_pointer_type(isa.triple());
393 let sret_arg = pos.ins().stack_addr(ptr_type, stack_slot, 0);
394 pos.func.dfg.append_inst_arg(call, sret_arg);
395
396 // The sret pointer might be returned by the signature as well. If so, we
397 // need to add it to the `call` instruction's results list.
398 //
399 // Additionally, when the sret is explicitly returned in this calling
400 // convention, then use it when loading the sret returns back into ssa
401 // values to avoid keeping the original `sret_arg` live and potentially
402 // having to do spills and fills.
403 let sret =
404 if pos.func.dfg.signatures[sig_ref].uses_special_return(ArgumentPurpose::StructReturn) {
405 pos.func.dfg.append_result(call, ptr_type)
406 } else {
407 sret_arg
408 };
409
410 // Finally, load each of the call's return values out of the sret stack
411 // slot.
412 pos.goto_after_inst(call);
413 let mut offset = 0;
414 for i in 0..old_ret_list.len(&pos.func.dfg.value_lists) {
415 if old_sig.returns[i].purpose != ArgumentPurpose::Normal {
416 continue;
417 }
418
419 let old_v = old_ret_list.get(i, &pos.func.dfg.value_lists).unwrap();
420 let ty = pos.func.dfg.value_type(old_v);
421 let mut legalized_ty = legalized_type_for_sret(ty);
422
423 offset = round_up_to_multiple_of_type_align(offset, legalized_ty);
424
425 let new_legalized_v =
426 pos.ins()
427 .load(legalized_ty, MemFlags::trusted(), sret, offset as i32);
428
429 // "Illegalize" the loaded value from the legalized type back to its
430 // original `ty`. This is basically the opposite of
431 // `legalize_type_for_sret_store`.
432 let mut new_v = new_legalized_v;
433 if ty.is_bool() {
434 legalized_ty = legalized_ty.as_bool_pedantic();
435 new_v = pos.ins().raw_bitcast(legalized_ty, new_v);
436
437 if ty.bits() < legalized_ty.bits() {
438 legalized_ty = ty;
439 new_v = pos.ins().breduce(legalized_ty, new_v);
440 }
441 }
442
443 pos.func.dfg.change_to_alias(old_v, new_v);
444
445 offset += legalized_ty.bytes();
446 }
447
448 pos.func.dfg.old_signatures[sig_ref] = Some(old_sig);
449 }
450
451 /// Compute original value of type `ty` from the legalized ABI arguments.
452 ///
453 /// The conversion is recursive, controlled by the `get_arg` closure which is called to retrieve an
454 /// ABI argument. It returns:
455 ///
456 /// - `Ok(arg)` if the requested type matches the next ABI argument.
457 /// - `Err(arg_type)` if further conversions are needed from the ABI argument `arg_type`.
458 ///
459 /// If the `into_result` value is provided, the converted result will be written into that value.
convert_from_abi<GetArg>( pos: &mut FuncCursor, ty: Type, into_result: Option<Value>, get_arg: &mut GetArg, ) -> Value where GetArg: FnMut(&mut Function, Type) -> Result<Value, AbiParam>,460 fn convert_from_abi<GetArg>(
461 pos: &mut FuncCursor,
462 ty: Type,
463 into_result: Option<Value>,
464 get_arg: &mut GetArg,
465 ) -> Value
466 where
467 GetArg: FnMut(&mut Function, Type) -> Result<Value, AbiParam>,
468 {
469 // Terminate the recursion when we get the desired type.
470 let arg_type = match get_arg(pos.func, ty) {
471 Ok(v) => {
472 debug_assert_eq!(pos.func.dfg.value_type(v), ty);
473 debug_assert_eq!(into_result, None);
474 return v;
475 }
476 Err(t) => t,
477 };
478
479 // Reconstruct how `ty` was legalized into the `arg_type` argument.
480 let conversion = legalize_abi_value(ty, &arg_type);
481
482 debug!("convert_from_abi({}): {:?}", ty, conversion);
483
484 // The conversion describes value to ABI argument. We implement the reverse conversion here.
485 match conversion {
486 // Construct a `ty` by concatenating two ABI integers.
487 ValueConversion::IntSplit => {
488 let abi_ty = ty.half_width().expect("Invalid type for conversion");
489 let lo = convert_from_abi(pos, abi_ty, None, get_arg);
490 let hi = convert_from_abi(pos, abi_ty, None, get_arg);
491 debug!(
492 "intsplit {}: {}, {}: {}",
493 lo,
494 pos.func.dfg.value_type(lo),
495 hi,
496 pos.func.dfg.value_type(hi)
497 );
498 pos.ins().with_results([into_result]).iconcat(lo, hi)
499 }
500 // Construct a `ty` by concatenating two halves of a vector.
501 ValueConversion::VectorSplit => {
502 let abi_ty = ty.half_vector().expect("Invalid type for conversion");
503 let lo = convert_from_abi(pos, abi_ty, None, get_arg);
504 let hi = convert_from_abi(pos, abi_ty, None, get_arg);
505 pos.ins().with_results([into_result]).vconcat(lo, hi)
506 }
507 // Construct a `ty` by bit-casting from an integer type.
508 ValueConversion::IntBits => {
509 debug_assert!(!ty.is_int());
510 let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
511 let arg = convert_from_abi(pos, abi_ty, None, get_arg);
512 pos.ins().with_results([into_result]).bitcast(ty, arg)
513 }
514 // ABI argument is a sign-extended version of the value we want.
515 ValueConversion::Sext(abi_ty) => {
516 let arg = convert_from_abi(pos, abi_ty, None, get_arg);
517 // TODO: Currently, we don't take advantage of the ABI argument being sign-extended.
518 // We could insert an `assert_sreduce` which would fold with a following `sextend` of
519 // this value.
520 pos.ins().with_results([into_result]).ireduce(ty, arg)
521 }
522 ValueConversion::Uext(abi_ty) => {
523 let arg = convert_from_abi(pos, abi_ty, None, get_arg);
524 // TODO: Currently, we don't take advantage of the ABI argument being sign-extended.
525 // We could insert an `assert_ureduce` which would fold with a following `uextend` of
526 // this value.
527 pos.ins().with_results([into_result]).ireduce(ty, arg)
528 }
529 // ABI argument is a pointer to the value we want.
530 ValueConversion::Pointer(abi_ty) => {
531 let arg = convert_from_abi(pos, abi_ty, None, get_arg);
532 pos.ins()
533 .with_results([into_result])
534 .load(ty, MemFlags::new(), arg, 0)
535 }
536 }
537 }
538
539 /// Convert `value` to match an ABI signature by inserting instructions at `pos`.
540 ///
541 /// This may require expanding the value to multiple ABI arguments. The conversion process is
542 /// recursive and controlled by the `put_arg` closure. When a candidate argument value is presented
543 /// to the closure, it will perform one of two actions:
544 ///
545 /// 1. If the suggested argument has an acceptable value type, consume it by adding it to the list
546 /// of arguments and return `Ok(())`.
547 /// 2. If the suggested argument doesn't have the right value type, don't change anything, but
548 /// return the `Err(AbiParam)` that is needed.
549 ///
convert_to_abi<PutArg>( pos: &mut FuncCursor, cfg: &ControlFlowGraph, value: Value, put_arg: &mut PutArg, ) where PutArg: FnMut(&mut Function, Value) -> Result<(), AbiParam>,550 fn convert_to_abi<PutArg>(
551 pos: &mut FuncCursor,
552 cfg: &ControlFlowGraph,
553 value: Value,
554 put_arg: &mut PutArg,
555 ) where
556 PutArg: FnMut(&mut Function, Value) -> Result<(), AbiParam>,
557 {
558 // Start by invoking the closure to either terminate the recursion or get the argument type
559 // we're trying to match.
560 let arg_type = match put_arg(pos.func, value) {
561 Ok(_) => return,
562 Err(t) => t,
563 };
564
565 let ty = pos.func.dfg.value_type(value);
566 match legalize_abi_value(ty, &arg_type) {
567 ValueConversion::IntSplit => {
568 let curpos = pos.position();
569 let srcloc = pos.srcloc();
570 let (lo, hi) = isplit(&mut pos.func, cfg, curpos, srcloc, value);
571 convert_to_abi(pos, cfg, lo, put_arg);
572 convert_to_abi(pos, cfg, hi, put_arg);
573 }
574 ValueConversion::VectorSplit => {
575 let curpos = pos.position();
576 let srcloc = pos.srcloc();
577 let (lo, hi) = vsplit(&mut pos.func, cfg, curpos, srcloc, value);
578 convert_to_abi(pos, cfg, lo, put_arg);
579 convert_to_abi(pos, cfg, hi, put_arg);
580 }
581 ValueConversion::IntBits => {
582 debug_assert!(!ty.is_int());
583 let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
584 let arg = pos.ins().bitcast(abi_ty, value);
585 convert_to_abi(pos, cfg, arg, put_arg);
586 }
587 ValueConversion::Sext(abi_ty) => {
588 let arg = pos.ins().sextend(abi_ty, value);
589 convert_to_abi(pos, cfg, arg, put_arg);
590 }
591 ValueConversion::Uext(abi_ty) => {
592 let arg = pos.ins().uextend(abi_ty, value);
593 convert_to_abi(pos, cfg, arg, put_arg);
594 }
595 ValueConversion::Pointer(abi_ty) => {
596 // Note: This conversion can only happen for call arguments,
597 // so we can allocate the value on stack safely.
598 let stack_slot = pos.func.create_stack_slot(StackSlotData {
599 kind: StackSlotKind::ExplicitSlot,
600 size: ty.bytes(),
601 offset: None,
602 });
603 let arg = pos.ins().stack_addr(abi_ty, stack_slot, 0);
604 pos.ins().store(MemFlags::new(), value, arg, 0);
605 convert_to_abi(pos, cfg, arg, put_arg);
606 }
607 }
608 }
609
610 /// Check if a sequence of arguments match a desired sequence of argument types.
check_arg_types(dfg: &DataFlowGraph, args: &[Value], types: &[AbiParam]) -> bool611 fn check_arg_types(dfg: &DataFlowGraph, args: &[Value], types: &[AbiParam]) -> bool {
612 args.len() == types.len()
613 && args.iter().zip(types.iter()).all(|(v, at)| {
614 if let ArgumentPurpose::StructArgument(_) = at.purpose {
615 true
616 } else {
617 dfg.value_type(*v) == at.value_type
618 }
619 })
620 }
621
622 /// Check if the arguments of the call `inst` match the signature.
623 ///
624 /// Returns `Ok(())` if the signature matches and no changes are needed, or `Err(sig_ref)` if the
625 /// signature doesn't match.
check_call_signature(dfg: &DataFlowGraph, inst: Inst) -> Result<(), SigRef>626 fn check_call_signature(dfg: &DataFlowGraph, inst: Inst) -> Result<(), SigRef> {
627 // Extract the signature and argument values.
628 let (sig_ref, args) = match dfg[inst].analyze_call(&dfg.value_lists) {
629 CallInfo::Direct(func, args) => (dfg.ext_funcs[func].signature, args),
630 CallInfo::Indirect(sig_ref, args) => (sig_ref, args),
631 CallInfo::NotACall => panic!("Expected call, got {:?}", dfg[inst]),
632 };
633 let sig = &dfg.signatures[sig_ref];
634
635 if check_arg_types(dfg, args, &sig.params[..])
636 && check_arg_types(dfg, dfg.inst_results(inst), &sig.returns[..])
637 {
638 // All types check out.
639 Ok(())
640 } else {
641 // Call types need fixing.
642 Err(sig_ref)
643 }
644 }
645
646 /// Check if the arguments of the return `inst` match the signature.
check_return_signature(dfg: &DataFlowGraph, inst: Inst, sig: &Signature) -> bool647 fn check_return_signature(dfg: &DataFlowGraph, inst: Inst, sig: &Signature) -> bool {
648 check_arg_types(dfg, dfg.inst_variable_args(inst), &sig.returns)
649 }
650
651 /// Insert ABI conversion code for the arguments to the call or return instruction at `pos`.
652 ///
653 /// - `abi_args` is the number of arguments that the ABI signature requires.
654 /// - `get_abi_type` is a closure that can provide the desired `AbiParam` for a given ABI
655 /// argument number in `0..abi_args`.
656 ///
legalize_inst_arguments<ArgType>( pos: &mut FuncCursor, cfg: &ControlFlowGraph, abi_args: usize, mut get_abi_type: ArgType, ) where ArgType: FnMut(&Function, usize) -> AbiParam,657 fn legalize_inst_arguments<ArgType>(
658 pos: &mut FuncCursor,
659 cfg: &ControlFlowGraph,
660 abi_args: usize,
661 mut get_abi_type: ArgType,
662 ) where
663 ArgType: FnMut(&Function, usize) -> AbiParam,
664 {
665 let inst = pos
666 .current_inst()
667 .expect("Cursor must point to a call instruction");
668
669 // Lift the value list out of the call instruction so we modify it.
670 let mut vlist = pos.func.dfg[inst]
671 .take_value_list()
672 .expect("Call must have a value list");
673
674 // The value list contains all arguments to the instruction, including the callee on an
675 // indirect call which isn't part of the call arguments that must match the ABI signature.
676 // Figure out how many fixed values are at the front of the list. We won't touch those.
677 let num_fixed_values = pos.func.dfg[inst]
678 .opcode()
679 .constraints()
680 .num_fixed_value_arguments();
681 let have_args = vlist.len(&pos.func.dfg.value_lists) - num_fixed_values;
682 if abi_args < have_args {
683 // This happens with multiple return values after we've legalized the
684 // signature but haven't legalized the return instruction yet. This
685 // legalization is handled in `handle_return_abi`.
686 pos.func.dfg[inst].put_value_list(vlist);
687 return;
688 }
689
690 // Grow the value list to the right size and shift all the existing arguments to the right.
691 // This lets us write the new argument values into the list without overwriting the old
692 // arguments.
693 //
694 // Before:
695 //
696 // <--> fixed_values
697 // <-----------> have_args
698 // [FFFFOOOOOOOOOOOOO]
699 //
700 // After grow_at():
701 //
702 // <--> fixed_values
703 // <-----------> have_args
704 // <------------------> abi_args
705 // [FFFF-------OOOOOOOOOOOOO]
706 // ^
707 // old_arg_offset
708 //
709 // After writing the new arguments:
710 //
711 // <--> fixed_values
712 // <------------------> abi_args
713 // [FFFFNNNNNNNNNNNNNNNNNNNN]
714 //
715 vlist.grow_at(
716 num_fixed_values,
717 abi_args - have_args,
718 &mut pos.func.dfg.value_lists,
719 );
720 let old_arg_offset = num_fixed_values + abi_args - have_args;
721
722 let mut abi_arg = 0;
723 for old_arg in 0..have_args {
724 let old_value = vlist
725 .get(old_arg_offset + old_arg, &pos.func.dfg.value_lists)
726 .unwrap();
727 let mut put_arg = |func: &mut Function, arg| {
728 let abi_type = get_abi_type(func, abi_arg);
729 let struct_argument = if let ArgumentPurpose::StructArgument(_) = abi_type.purpose {
730 true
731 } else {
732 false
733 };
734 if func.dfg.value_type(arg) == abi_type.value_type || struct_argument {
735 // This is the argument type we need.
736 vlist.as_mut_slice(&mut func.dfg.value_lists)[num_fixed_values + abi_arg] = arg;
737 abi_arg += 1;
738 Ok(())
739 } else {
740 Err(abi_type)
741 }
742 };
743 convert_to_abi(pos, cfg, old_value, &mut put_arg);
744 }
745
746 // Put the modified value list back.
747 pos.func.dfg[inst].put_value_list(vlist);
748 }
749
750 /// Ensure that the `ty` being returned is a type that can be loaded and stored
751 /// (potentially after another narrowing legalization) from memory, since it
752 /// will go into the `sret` space.
legalized_type_for_sret(ty: Type) -> Type753 fn legalized_type_for_sret(ty: Type) -> Type {
754 if ty.is_bool() {
755 let bits = std::cmp::max(8, ty.bits());
756 Type::int(bits).unwrap()
757 } else {
758 ty
759 }
760 }
761
762 /// Insert any legalization code required to ensure that `val` can be stored
763 /// into the `sret` memory. Returns the (potentially new, potentially
764 /// unmodified) legalized value and its type.
legalize_type_for_sret_store(pos: &mut FuncCursor, val: Value, ty: Type) -> (Value, Type)765 fn legalize_type_for_sret_store(pos: &mut FuncCursor, val: Value, ty: Type) -> (Value, Type) {
766 if ty.is_bool() {
767 let bits = std::cmp::max(8, ty.bits());
768 let ty = Type::int(bits).unwrap();
769 let val = pos.ins().bint(ty, val);
770 (val, ty)
771 } else {
772 (val, ty)
773 }
774 }
775
776 /// Insert ABI conversion code before and after the call instruction at `pos`.
777 ///
778 /// Instructions inserted before the call will compute the appropriate ABI values for the
779 /// callee's new ABI-legalized signature. The function call arguments are rewritten in place to
780 /// match the new signature.
781 ///
782 /// Instructions will be inserted after the call to convert returned ABI values back to the
783 /// original return values. The call's result values will be adapted to match the new signature.
784 ///
785 /// Returns `true` if any instructions were inserted.
handle_call_abi( isa: &dyn TargetIsa, mut inst: Inst, func: &mut Function, cfg: &ControlFlowGraph, ) -> bool786 pub fn handle_call_abi(
787 isa: &dyn TargetIsa,
788 mut inst: Inst,
789 func: &mut Function,
790 cfg: &ControlFlowGraph,
791 ) -> bool {
792 let pos = &mut FuncCursor::new(func).at_inst(inst);
793 pos.use_srcloc(inst);
794
795 // Start by checking if the argument types already match the signature.
796 let sig_ref = match check_call_signature(&pos.func.dfg, inst) {
797 Ok(_) => return spill_call_arguments(pos, isa),
798 Err(s) => s,
799 };
800
801 let sig = &pos.func.dfg.signatures[sig_ref];
802 let old_sig = &pos.func.dfg.old_signatures[sig_ref];
803
804 if sig.uses_struct_return_param()
805 && old_sig
806 .as_ref()
807 .map_or(false, |s| !s.uses_struct_return_param())
808 {
809 legalize_sret_call(isa, pos, sig_ref, inst);
810 } else {
811 if !pos.func.dfg.signatures[sig_ref].returns.is_empty() {
812 inst = legalize_inst_results(pos, |func, abi_res| {
813 func.dfg.signatures[sig_ref].returns[abi_res]
814 });
815 }
816 }
817
818 // Go back and fix the call arguments to match the ABI signature.
819 pos.goto_inst(inst);
820 let abi_args = pos.func.dfg.signatures[sig_ref].params.len();
821 legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
822 func.dfg.signatures[sig_ref].params[abi_arg]
823 });
824
825 debug_assert!(
826 check_call_signature(&pos.func.dfg, inst).is_ok(),
827 "Signature still wrong: {}, {}{}",
828 pos.func.dfg.display_inst(inst, None),
829 sig_ref,
830 pos.func.dfg.signatures[sig_ref]
831 );
832
833 // Go back and insert spills for any stack arguments.
834 pos.goto_inst(inst);
835 spill_call_arguments(pos, isa);
836
837 // Yes, we changed stuff.
838 true
839 }
840
841 /// Insert ABI conversion code before and after the return instruction at `inst`.
842 ///
843 /// Return `true` if any instructions were inserted.
handle_return_abi(inst: Inst, func: &mut Function, cfg: &ControlFlowGraph) -> bool844 pub fn handle_return_abi(inst: Inst, func: &mut Function, cfg: &ControlFlowGraph) -> bool {
845 // Check if the returned types already match the signature.
846 if check_return_signature(&func.dfg, inst, &func.signature) {
847 return false;
848 }
849
850 // Count the special-purpose return values (`link`, `sret`, and `vmctx`) that were appended to
851 // the legalized signature.
852 let special_args = func
853 .signature
854 .returns
855 .iter()
856 .rev()
857 .take_while(|&rt| {
858 rt.purpose == ArgumentPurpose::Link
859 || rt.purpose == ArgumentPurpose::StructReturn
860 || rt.purpose == ArgumentPurpose::VMContext
861 })
862 .count();
863 let abi_args = func.signature.returns.len() - special_args;
864
865 let pos = &mut FuncCursor::new(func).at_inst(inst);
866 pos.use_srcloc(inst);
867
868 legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
869 let arg = func.signature.returns[abi_arg];
870 debug_assert!(
871 !arg.legalized_to_pointer,
872 "Return value cannot be legalized to pointer"
873 );
874 arg
875 });
876 // Append special return arguments for any `sret`, `link`, and `vmctx` return values added to
877 // the legalized signature. These values should simply be propagated from the entry block
878 // arguments.
879 if special_args > 0 {
880 debug!(
881 "Adding {} special-purpose arguments to {}",
882 special_args,
883 pos.func.dfg.display_inst(inst, None)
884 );
885 let mut vlist = pos.func.dfg[inst].take_value_list().unwrap();
886 let mut sret = None;
887
888 for arg in &pos.func.signature.returns[abi_args..] {
889 match arg.purpose {
890 ArgumentPurpose::Link
891 | ArgumentPurpose::StructReturn
892 | ArgumentPurpose::VMContext => {}
893 ArgumentPurpose::Normal => panic!("unexpected return value {}", arg),
894 _ => panic!("Unsupported special purpose return value {}", arg),
895 }
896 // A `link`/`sret`/`vmctx` return value can only appear in a signature that has a
897 // unique matching argument. They are appended at the end, so search the signature from
898 // the end.
899 let idx = pos
900 .func
901 .signature
902 .params
903 .iter()
904 .rposition(|t| t.purpose == arg.purpose)
905 .expect("No matching special purpose argument.");
906 // Get the corresponding entry block value and add it to the return instruction's
907 // arguments.
908 let val = pos
909 .func
910 .dfg
911 .block_params(pos.func.layout.entry_block().unwrap())[idx];
912 debug_assert_eq!(pos.func.dfg.value_type(val), arg.value_type);
913 vlist.push(val, &mut pos.func.dfg.value_lists);
914
915 if let ArgumentPurpose::StructReturn = arg.purpose {
916 sret = Some(val);
917 }
918 }
919
920 // Store all the regular returns into the retptr space and remove them
921 // from the `return` instruction's value list.
922 if let Some(sret) = sret {
923 let mut offset = 0;
924 let num_regular_rets = vlist.len(&pos.func.dfg.value_lists) - special_args;
925 for i in 0..num_regular_rets {
926 debug_assert_eq!(
927 pos.func.old_signature.as_ref().unwrap().returns[i].purpose,
928 ArgumentPurpose::Normal,
929 );
930
931 // The next return value to process is always at `0`, since the
932 // list is emptied as we iterate.
933 let v = vlist.get(0, &pos.func.dfg.value_lists).unwrap();
934 let ty = pos.func.dfg.value_type(v);
935 let (v, ty) = legalize_type_for_sret_store(pos, v, ty);
936
937 let size = ty.bytes();
938 offset = round_up_to_multiple_of_type_align(offset, ty);
939
940 pos.ins().store(MemFlags::trusted(), v, sret, offset as i32);
941 vlist.remove(0, &mut pos.func.dfg.value_lists);
942
943 offset += size;
944 }
945 }
946 pos.func.dfg[inst].put_value_list(vlist);
947 }
948
949 debug_assert_eq!(
950 pos.func.dfg.inst_variable_args(inst).len(),
951 abi_args + special_args
952 );
953 debug_assert!(
954 check_return_signature(&pos.func.dfg, inst, &pos.func.signature),
955 "Signature still wrong: {} / signature {}",
956 pos.func.dfg.display_inst(inst, None),
957 pos.func.signature
958 );
959
960 // Yes, we changed stuff.
961 true
962 }
963
round_up_to_multiple_of_type_align(bytes: u32, ty: Type) -> u32964 fn round_up_to_multiple_of_type_align(bytes: u32, ty: Type) -> u32 {
965 // We don't have a dedicated alignment for types, so assume they are
966 // size-aligned.
967 let align = ty.bytes();
968 round_up_to_multiple_of_pow2(bytes, align)
969 }
970
971 /// Round `n` up to the next multiple of `to` that is greater than or equal to
972 /// `n`.
973 ///
974 /// `to` must be a power of two and greater than zero.
975 ///
976 /// This is useful for rounding an offset or pointer up to some type's required
977 /// alignment.
round_up_to_multiple_of_pow2(n: u32, to: u32) -> u32978 fn round_up_to_multiple_of_pow2(n: u32, to: u32) -> u32 {
979 debug_assert!(to > 0);
980 debug_assert!(to.is_power_of_two());
981
982 // The simple version of this function is
983 //
984 // (n + to - 1) / to * to
985 //
986 // Consider the numerator: `n + to - 1`. This is ensuring that if there is
987 // any remainder for `n / to`, then the result of the division is one
988 // greater than `n / to`, and that otherwise we get exactly the same result
989 // as `n / to` due to integer division rounding off the remainder. In other
990 // words, we only round up if `n` is not aligned to `to`.
991 //
992 // However, we know `to` is a power of two, and therefore `anything / to` is
993 // equivalent to `anything >> log2(to)` and `anything * to` is equivalent to
994 // `anything << log2(to)`. We can therefore rewrite our simplified function
995 // into the following:
996 //
997 // (n + to - 1) >> log2(to) << log2(to)
998 //
999 // But shifting a value right by some number of bits `b` and then shifting
1000 // it left by that same number of bits `b` is equivalent to clearing the
1001 // bottom `b` bits of the number. We can clear the bottom `b` bits of a
1002 // number by bit-wise and'ing the number with the bit-wise not of `2^b - 1`.
1003 // Plugging this into our function and simplifying, we get:
1004 //
1005 // (n + to - 1) >> log2(to) << log2(to)
1006 // = (n + to - 1) & !(2^log2(to) - 1)
1007 // = (n + to - 1) & !(to - 1)
1008 //
1009 // And now we have the final version of this function!
1010
1011 (n + to - 1) & !(to - 1)
1012 }
1013
1014 /// Assign stack slots to incoming function parameters on the stack.
1015 ///
1016 /// Values that are passed into the function on the stack must be assigned to an `IncomingArg`
1017 /// stack slot already during legalization.
spill_entry_params(func: &mut Function, entry: Block)1018 fn spill_entry_params(func: &mut Function, entry: Block) {
1019 for (abi, &arg) in func
1020 .signature
1021 .params
1022 .iter()
1023 .zip(func.dfg.block_params(entry))
1024 {
1025 if let ArgumentPurpose::StructArgument(_) = abi.purpose {
1026 // Location has already been assigned during legalization.
1027 } else if let ArgumentLoc::Stack(offset) = abi.location {
1028 let ss = func
1029 .stack_slots
1030 .make_incoming_arg(abi.value_type.bytes(), offset);
1031 func.locations[arg] = ValueLoc::Stack(ss);
1032 }
1033 }
1034 }
1035
1036 /// Assign stack slots to outgoing function arguments on the stack.
1037 ///
1038 /// Values that are passed to a called function on the stack must be assigned to a matching
1039 /// `OutgoingArg` stack slot. The assignment must happen immediately before the call.
1040 ///
1041 /// TODO: The outgoing stack slots can be written a bit earlier, as long as there are no branches
1042 /// or calls between writing the stack slots and the call instruction. Writing the slots earlier
1043 /// could help reduce register pressure before the call.
spill_call_arguments(pos: &mut FuncCursor, isa: &dyn TargetIsa) -> bool1044 fn spill_call_arguments(pos: &mut FuncCursor, isa: &dyn TargetIsa) -> bool {
1045 let inst = pos
1046 .current_inst()
1047 .expect("Cursor must point to a call instruction");
1048 let sig_ref = pos
1049 .func
1050 .dfg
1051 .call_signature(inst)
1052 .expect("Call instruction expected.");
1053
1054 // Start by building a list of stack slots and arguments to be replaced.
1055 // This requires borrowing `pos.func.dfg`, so we can't change anything.
1056 let arglist = {
1057 let locations = &pos.func.locations;
1058 let stack_slots = &mut pos.func.stack_slots;
1059 pos.func
1060 .dfg
1061 .inst_variable_args(inst)
1062 .iter()
1063 .zip(&pos.func.dfg.signatures[sig_ref].params)
1064 .enumerate()
1065 .filter_map(|(idx, (&arg, abi))| {
1066 match abi.location {
1067 ArgumentLoc::Stack(offset) => {
1068 // Assign `arg` to a new stack slot, unless it's already in the correct
1069 // slot. The legalization needs to be idempotent, so we should see a
1070 // correct outgoing slot on the second pass.
1071 let (ss, size) = match abi.purpose {
1072 ArgumentPurpose::StructArgument(size) => {
1073 (stack_slots.get_outgoing_arg(size, offset), Some(size))
1074 }
1075 _ => (
1076 stack_slots.get_outgoing_arg(abi.value_type.bytes(), offset),
1077 None,
1078 ),
1079 };
1080 if locations[arg] != ValueLoc::Stack(ss) {
1081 Some((idx, arg, ss, size))
1082 } else {
1083 None
1084 }
1085 }
1086 _ => None,
1087 }
1088 })
1089 .collect::<Vec<_>>()
1090 };
1091
1092 if arglist.is_empty() {
1093 return false;
1094 }
1095
1096 let mut libc_memcpy = None;
1097 let mut import_memcpy = |func: &mut Function, pointer_type| {
1098 if let Some(libc_memcpy) = libc_memcpy {
1099 return libc_memcpy;
1100 }
1101
1102 let signature = {
1103 let mut s = Signature::new(isa.default_call_conv());
1104 s.params.push(AbiParam::new(pointer_type));
1105 s.params.push(AbiParam::new(pointer_type));
1106 // The last argument of `memcpy` is a `size_t`. This is the same size as a pointer on
1107 // all architectures we are interested in.
1108 s.params.push(AbiParam::new(pointer_type));
1109 legalize_libcall_signature(&mut s, isa);
1110 func.import_signature(s)
1111 };
1112
1113 let func = func.import_function(ExtFuncData {
1114 name: ExternalName::LibCall(LibCall::Memcpy),
1115 signature,
1116 colocated: false,
1117 });
1118 libc_memcpy = Some(func);
1119 func
1120 };
1121
1122 // Insert the spill instructions and rewrite call arguments.
1123 for (idx, arg, ss, size) in arglist {
1124 let stack_val = if let Some(size) = size {
1125 // Struct argument
1126 let pointer_type = pos.func.dfg.value_type(arg);
1127 let src = arg;
1128 let dest = pos.ins().stack_addr(pointer_type, ss, 0);
1129 let size = pos.ins().iconst(pointer_type, i64::from(size));
1130
1131 let libc_memcpy = import_memcpy(pos.func, pointer_type);
1132 pos.ins().call(libc_memcpy, &[dest, src, size]);
1133 pos.ins().dummy_sarg_t()
1134 } else {
1135 // Non struct argument
1136 pos.ins().spill(arg)
1137 };
1138 pos.func.locations[stack_val] = ValueLoc::Stack(ss);
1139 pos.func.dfg.inst_variable_args_mut(inst)[idx] = stack_val;
1140 }
1141
1142 // We changed stuff.
1143 true
1144 }
1145
1146 #[cfg(test)]
1147 mod tests {
1148 use super::round_up_to_multiple_of_pow2;
1149
1150 #[test]
round_up_to_multiple_of_pow2_works()1151 fn round_up_to_multiple_of_pow2_works() {
1152 for (n, to, expected) in vec![
1153 (0, 1, 0),
1154 (1, 1, 1),
1155 (2, 1, 2),
1156 (0, 2, 0),
1157 (1, 2, 2),
1158 (2, 2, 2),
1159 (3, 2, 4),
1160 (0, 4, 0),
1161 (1, 4, 4),
1162 (2, 4, 4),
1163 (3, 4, 4),
1164 (4, 4, 4),
1165 (5, 4, 8),
1166 ] {
1167 let actual = round_up_to_multiple_of_pow2(n, to);
1168 assert_eq!(
1169 actual, expected,
1170 "round_up_to_multiple_of_pow2(n = {}, to = {}) = {} (expected {})",
1171 n, to, actual, expected
1172 );
1173 }
1174 }
1175 }
1176