use crate::abi::{legalize_abi_value, ValueConversion};
use crate::cursor::{Cursor, FuncCursor};
use crate::flowgraph::ControlFlowGraph;
use crate::ir::instructions::CallInfo;
use crate::ir::{
AbiParam, ArgumentLoc, ArgumentPurpose, Block, DataFlowGraph, ExtFuncData, ExternalName,
Function, Inst, InstBuilder, LibCall, MemFlags, SigRef, Signature, StackSlotData,
StackSlotKind, Type, Value, ValueLoc,
};
use crate::isa::TargetIsa;
use crate::legalizer::split::{isplit, vsplit};
use alloc::borrow::Cow;
use alloc::vec::Vec;
use core::mem;
use cranelift_entity::EntityList;
use log::debug;
pub fn legalize_signatures(func: &mut Function, isa: &dyn TargetIsa) {
if let Some(new) = legalize_signature(&func.signature, true, isa) {
let old = mem::replace(&mut func.signature, new);
func.old_signature = Some(old);
}
for (sig_ref, sig_data) in func.dfg.signatures.iter_mut() {
if let Some(new) = legalize_signature(sig_data, false, isa) {
let old = mem::replace(sig_data, new);
func.dfg.old_signatures[sig_ref] = Some(old);
}
}
if let Some(entry) = func.layout.entry_block() {
legalize_entry_params(func, entry);
spill_entry_params(func, entry);
}
}
pub fn legalize_libcall_signature(signature: &mut Signature, isa: &dyn TargetIsa) {
if let Some(s) = legalize_signature(signature, false, isa) {
*signature = s;
}
}
fn legalize_signature(
signature: &Signature,
current: bool,
isa: &dyn TargetIsa,
) -> Option<Signature> {
let mut cow = Cow::Borrowed(signature);
isa.legalize_signature(&mut cow, current);
match cow {
Cow::Borrowed(_) => None,
Cow::Owned(s) => Some(s),
}
}
fn legalize_entry_params(func: &mut Function, entry: Block) {
let mut has_sret = false;
let mut has_link = false;
let mut has_vmctx = false;
let mut has_sigid = false;
let mut has_stack_limit = false;
let mut pos = FuncCursor::new(func).at_first_inst(entry);
let mut abi_arg = 0;
let block_params = pos.func.dfg.detach_block_params(entry);
let mut old_arg = 0;
while let Some(arg) = block_params.get(old_arg, &pos.func.dfg.value_lists) {
old_arg += 1;
let abi_type = pos.func.signature.params[abi_arg];
let arg_type = pos.func.dfg.value_type(arg);
if let ArgumentPurpose::StructArgument(size) = abi_type.purpose {
let offset = if let ArgumentLoc::Stack(offset) = abi_type.location {
offset
} else {
unreachable!("StructArgument must already have a Stack ArgumentLoc assigned");
};
let ss = pos.func.stack_slots.make_incoming_arg(size, offset);
let struct_arg = pos.ins().stack_addr(arg_type, ss, 0);
pos.func.dfg.change_to_alias(arg, struct_arg);
let dummy = pos
.func
.dfg
.append_block_param(entry, crate::ir::types::SARG_T);
pos.func.locations[dummy] = ValueLoc::Stack(ss);
abi_arg += 1;
continue;
}
if arg_type == abi_type.value_type {
pos.func.dfg.attach_block_param(entry, arg);
match abi_type.purpose {
ArgumentPurpose::Normal => {}
ArgumentPurpose::StructArgument(_) => unreachable!("Handled above"),
ArgumentPurpose::FramePointer => {}
ArgumentPurpose::CalleeSaved => {}
ArgumentPurpose::StructReturn => {
debug_assert!(!has_sret, "Multiple sret arguments found");
has_sret = true;
}
ArgumentPurpose::VMContext => {
debug_assert!(!has_vmctx, "Multiple vmctx arguments found");
has_vmctx = true;
}
ArgumentPurpose::SignatureId => {
debug_assert!(!has_sigid, "Multiple sigid arguments found");
has_sigid = true;
}
ArgumentPurpose::StackLimit => {
debug_assert!(!has_stack_limit, "Multiple stack_limit arguments found");
has_stack_limit = true;
}
ArgumentPurpose::Link => panic!("Unexpected link arg {}", abi_type),
ArgumentPurpose::CallerTLS | ArgumentPurpose::CalleeTLS => {}
}
abi_arg += 1;
} else {
let mut get_arg = |func: &mut Function, ty| {
let abi_type = func.signature.params[abi_arg];
debug_assert_eq!(
abi_type.purpose,
ArgumentPurpose::Normal,
"Can't legalize special-purpose argument"
);
if ty == abi_type.value_type {
abi_arg += 1;
Ok(func.dfg.append_block_param(entry, ty))
} else {
Err(abi_type)
}
};
let converted = convert_from_abi(&mut pos, arg_type, Some(arg), &mut get_arg);
debug_assert_eq!(pos.func.dfg.resolve_aliases(arg), converted);
}
}
for &arg in &pos.func.signature.params[abi_arg..] {
match arg.purpose {
ArgumentPurpose::Normal | ArgumentPurpose::StructArgument(_) => {
panic!("Leftover arg: {}", arg);
}
ArgumentPurpose::FramePointer | ArgumentPurpose::CalleeSaved => {
panic!("Premature callee-saved arg {}", arg);
}
ArgumentPurpose::Link => {
debug_assert!(!has_link, "Multiple link parameters found");
has_link = true;
}
ArgumentPurpose::StructReturn => {
debug_assert!(!has_sret, "Multiple sret parameters found");
has_sret = true;
}
ArgumentPurpose::VMContext => {
debug_assert!(!has_vmctx, "Multiple vmctx parameters found");
has_vmctx = true;
}
ArgumentPurpose::SignatureId => {
debug_assert!(!has_sigid, "Multiple sigid parameters found");
has_sigid = true;
}
ArgumentPurpose::StackLimit => {
debug_assert!(!has_stack_limit, "Multiple stack_limit parameters found");
has_stack_limit = true;
}
ArgumentPurpose::CallerTLS | ArgumentPurpose::CalleeTLS => {}
}
pos.func.dfg.append_block_param(entry, arg.value_type);
}
}
fn legalize_inst_results<ResType>(pos: &mut FuncCursor, mut get_abi_type: ResType) -> Inst
where
ResType: FnMut(&Function, usize) -> AbiParam,
{
let call = pos
.current_inst()
.expect("Cursor must point to a call instruction");
debug_assert_eq!(
pos.func.dfg[call]
.opcode()
.constraints()
.num_fixed_results(),
0,
"Fixed results on calls not supported"
);
let results = pos.func.dfg.detach_results(call);
let mut next_res = 0;
let mut abi_res = 0;
pos.next_inst();
while let Some(res) = results.get(next_res, &pos.func.dfg.value_lists) {
next_res += 1;
let res_type = pos.func.dfg.value_type(res);
if res_type == get_abi_type(pos.func, abi_res).value_type {
pos.func.dfg.attach_result(call, res);
abi_res += 1;
} else {
let mut get_res = |func: &mut Function, ty| {
let abi_type = get_abi_type(func, abi_res);
if ty == abi_type.value_type {
let last_res = func.dfg.append_result(call, ty);
abi_res += 1;
Ok(last_res)
} else {
Err(abi_type)
}
};
let v = convert_from_abi(pos, res_type, Some(res), &mut get_res);
debug_assert_eq!(pos.func.dfg.resolve_aliases(res), v);
}
}
call
}
fn assert_is_valid_sret_legalization(
old_ret_list: &EntityList<Value>,
old_sig: &Signature,
new_sig: &Signature,
pos: &FuncCursor,
) {
debug_assert_eq!(
old_sig.returns.len(),
old_ret_list.len(&pos.func.dfg.value_lists)
);
let old_special_params: Vec<_> = old_sig
.params
.iter()
.filter(|r| r.purpose != ArgumentPurpose::Normal)
.collect();
let new_special_params: Vec<_> = new_sig
.params
.iter()
.filter(|r| r.purpose != ArgumentPurpose::Normal)
.collect();
debug_assert_eq!(old_special_params.len() + 1, new_special_params.len());
debug_assert!(old_special_params
.iter()
.zip(&new_special_params)
.all(|(old, new)| old.purpose == new.purpose));
debug_assert_eq!(
new_special_params.last().unwrap().purpose,
ArgumentPurpose::StructReturn
);
let old_special_returns: Vec<_> = old_sig
.returns
.iter()
.filter(|r| r.purpose != ArgumentPurpose::Normal)
.collect();
let new_special_returns: Vec<_> = new_sig
.returns
.iter()
.filter(|r| r.purpose != ArgumentPurpose::Normal)
.collect();
debug_assert!(old_special_returns
.iter()
.zip(&new_special_returns)
.all(|(old, new)| old.purpose == new.purpose));
debug_assert!(
old_special_returns.len() == new_special_returns.len()
|| (old_special_returns.len() + 1 == new_special_returns.len()
&& new_special_returns.last().unwrap().purpose == ArgumentPurpose::StructReturn)
);
}
fn legalize_sret_call(isa: &dyn TargetIsa, pos: &mut FuncCursor, sig_ref: SigRef, call: Inst) {
let old_ret_list = pos.func.dfg.detach_results(call);
let old_sig = pos.func.dfg.old_signatures[sig_ref]
.take()
.expect("must have an old signature when using an `sret` parameter");
if cfg!(debug_assertions) {
assert_is_valid_sret_legalization(
&old_ret_list,
&old_sig,
&pos.func.dfg.signatures[sig_ref],
&pos,
);
}
let mut sret_slot_size = 0;
for (i, ret) in old_sig.returns.iter().enumerate() {
let v = old_ret_list.get(i, &pos.func.dfg.value_lists).unwrap();
let ty = pos.func.dfg.value_type(v);
if ret.purpose == ArgumentPurpose::Normal {
debug_assert_eq!(ret.location, ArgumentLoc::Unassigned);
let ty = legalized_type_for_sret(ty);
let size = ty.bytes();
sret_slot_size = round_up_to_multiple_of_type_align(sret_slot_size, ty) + size;
} else {
let new_v = pos.func.dfg.append_result(call, ty);
pos.func.dfg.change_to_alias(v, new_v);
}
}
let stack_slot = pos.func.stack_slots.push(StackSlotData {
kind: StackSlotKind::StructReturnSlot,
size: sret_slot_size,
offset: None,
});
let ptr_type = Type::triple_pointer_type(isa.triple());
let sret_arg = pos.ins().stack_addr(ptr_type, stack_slot, 0);
pos.func.dfg.append_inst_arg(call, sret_arg);
let sret =
if pos.func.dfg.signatures[sig_ref].uses_special_return(ArgumentPurpose::StructReturn) {
pos.func.dfg.append_result(call, ptr_type)
} else {
sret_arg
};
pos.goto_after_inst(call);
let mut offset = 0;
for i in 0..old_ret_list.len(&pos.func.dfg.value_lists) {
if old_sig.returns[i].purpose != ArgumentPurpose::Normal {
continue;
}
let old_v = old_ret_list.get(i, &pos.func.dfg.value_lists).unwrap();
let ty = pos.func.dfg.value_type(old_v);
let mut legalized_ty = legalized_type_for_sret(ty);
offset = round_up_to_multiple_of_type_align(offset, legalized_ty);
let new_legalized_v =
pos.ins()
.load(legalized_ty, MemFlags::trusted(), sret, offset as i32);
let mut new_v = new_legalized_v;
if ty.is_bool() {
legalized_ty = legalized_ty.as_bool_pedantic();
new_v = pos.ins().raw_bitcast(legalized_ty, new_v);
if ty.bits() < legalized_ty.bits() {
legalized_ty = ty;
new_v = pos.ins().breduce(legalized_ty, new_v);
}
}
pos.func.dfg.change_to_alias(old_v, new_v);
offset += legalized_ty.bytes();
}
pos.func.dfg.old_signatures[sig_ref] = Some(old_sig);
}
fn convert_from_abi<GetArg>(
pos: &mut FuncCursor,
ty: Type,
into_result: Option<Value>,
get_arg: &mut GetArg,
) -> Value
where
GetArg: FnMut(&mut Function, Type) -> Result<Value, AbiParam>,
{
let arg_type = match get_arg(pos.func, ty) {
Ok(v) => {
debug_assert_eq!(pos.func.dfg.value_type(v), ty);
debug_assert_eq!(into_result, None);
return v;
}
Err(t) => t,
};
let conversion = legalize_abi_value(ty, &arg_type);
debug!("convert_from_abi({}): {:?}", ty, conversion);
match conversion {
ValueConversion::IntSplit => {
let abi_ty = ty.half_width().expect("Invalid type for conversion");
let lo = convert_from_abi(pos, abi_ty, None, get_arg);
let hi = convert_from_abi(pos, abi_ty, None, get_arg);
debug!(
"intsplit {}: {}, {}: {}",
lo,
pos.func.dfg.value_type(lo),
hi,
pos.func.dfg.value_type(hi)
);
pos.ins().with_results([into_result]).iconcat(lo, hi)
}
ValueConversion::VectorSplit => {
let abi_ty = ty.half_vector().expect("Invalid type for conversion");
let lo = convert_from_abi(pos, abi_ty, None, get_arg);
let hi = convert_from_abi(pos, abi_ty, None, get_arg);
pos.ins().with_results([into_result]).vconcat(lo, hi)
}
ValueConversion::IntBits => {
debug_assert!(!ty.is_int());
let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
pos.ins().with_results([into_result]).bitcast(ty, arg)
}
ValueConversion::Sext(abi_ty) => {
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
pos.ins().with_results([into_result]).ireduce(ty, arg)
}
ValueConversion::Uext(abi_ty) => {
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
pos.ins().with_results([into_result]).ireduce(ty, arg)
}
ValueConversion::Pointer(abi_ty) => {
let arg = convert_from_abi(pos, abi_ty, None, get_arg);
pos.ins()
.with_results([into_result])
.load(ty, MemFlags::new(), arg, 0)
}
}
}
fn convert_to_abi<PutArg>(
pos: &mut FuncCursor,
cfg: &ControlFlowGraph,
value: Value,
put_arg: &mut PutArg,
) where
PutArg: FnMut(&mut Function, Value) -> Result<(), AbiParam>,
{
let arg_type = match put_arg(pos.func, value) {
Ok(_) => return,
Err(t) => t,
};
let ty = pos.func.dfg.value_type(value);
match legalize_abi_value(ty, &arg_type) {
ValueConversion::IntSplit => {
let curpos = pos.position();
let srcloc = pos.srcloc();
let (lo, hi) = isplit(&mut pos.func, cfg, curpos, srcloc, value);
convert_to_abi(pos, cfg, lo, put_arg);
convert_to_abi(pos, cfg, hi, put_arg);
}
ValueConversion::VectorSplit => {
let curpos = pos.position();
let srcloc = pos.srcloc();
let (lo, hi) = vsplit(&mut pos.func, cfg, curpos, srcloc, value);
convert_to_abi(pos, cfg, lo, put_arg);
convert_to_abi(pos, cfg, hi, put_arg);
}
ValueConversion::IntBits => {
debug_assert!(!ty.is_int());
let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
let arg = pos.ins().bitcast(abi_ty, value);
convert_to_abi(pos, cfg, arg, put_arg);
}
ValueConversion::Sext(abi_ty) => {
let arg = pos.ins().sextend(abi_ty, value);
convert_to_abi(pos, cfg, arg, put_arg);
}
ValueConversion::Uext(abi_ty) => {
let arg = pos.ins().uextend(abi_ty, value);
convert_to_abi(pos, cfg, arg, put_arg);
}
ValueConversion::Pointer(abi_ty) => {
let stack_slot = pos.func.create_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
size: ty.bytes(),
offset: None,
});
let arg = pos.ins().stack_addr(abi_ty, stack_slot, 0);
pos.ins().store(MemFlags::new(), value, arg, 0);
convert_to_abi(pos, cfg, arg, put_arg);
}
}
}
fn check_arg_types(dfg: &DataFlowGraph, args: &[Value], types: &[AbiParam]) -> bool {
args.len() == types.len()
&& args.iter().zip(types.iter()).all(|(v, at)| {
if let ArgumentPurpose::StructArgument(_) = at.purpose {
true
} else {
dfg.value_type(*v) == at.value_type
}
})
}
fn check_call_signature(dfg: &DataFlowGraph, inst: Inst) -> Result<(), SigRef> {
let (sig_ref, args) = match dfg[inst].analyze_call(&dfg.value_lists) {
CallInfo::Direct(func, args) => (dfg.ext_funcs[func].signature, args),
CallInfo::Indirect(sig_ref, args) => (sig_ref, args),
CallInfo::NotACall => panic!("Expected call, got {:?}", dfg[inst]),
};
let sig = &dfg.signatures[sig_ref];
if check_arg_types(dfg, args, &sig.params[..])
&& check_arg_types(dfg, dfg.inst_results(inst), &sig.returns[..])
{
Ok(())
} else {
Err(sig_ref)
}
}
fn check_return_signature(dfg: &DataFlowGraph, inst: Inst, sig: &Signature) -> bool {
check_arg_types(dfg, dfg.inst_variable_args(inst), &sig.returns)
}
fn legalize_inst_arguments<ArgType>(
pos: &mut FuncCursor,
cfg: &ControlFlowGraph,
abi_args: usize,
mut get_abi_type: ArgType,
) where
ArgType: FnMut(&Function, usize) -> AbiParam,
{
let inst = pos
.current_inst()
.expect("Cursor must point to a call instruction");
let mut vlist = pos.func.dfg[inst]
.take_value_list()
.expect("Call must have a value list");
let num_fixed_values = pos.func.dfg[inst]
.opcode()
.constraints()
.num_fixed_value_arguments();
let have_args = vlist.len(&pos.func.dfg.value_lists) - num_fixed_values;
if abi_args < have_args {
pos.func.dfg[inst].put_value_list(vlist);
return;
}
vlist.grow_at(
num_fixed_values,
abi_args - have_args,
&mut pos.func.dfg.value_lists,
);
let old_arg_offset = num_fixed_values + abi_args - have_args;
let mut abi_arg = 0;
for old_arg in 0..have_args {
let old_value = vlist
.get(old_arg_offset + old_arg, &pos.func.dfg.value_lists)
.unwrap();
let mut put_arg = |func: &mut Function, arg| {
let abi_type = get_abi_type(func, abi_arg);
let struct_argument = if let ArgumentPurpose::StructArgument(_) = abi_type.purpose {
true
} else {
false
};
if func.dfg.value_type(arg) == abi_type.value_type || struct_argument {
vlist.as_mut_slice(&mut func.dfg.value_lists)[num_fixed_values + abi_arg] = arg;
abi_arg += 1;
Ok(())
} else {
Err(abi_type)
}
};
convert_to_abi(pos, cfg, old_value, &mut put_arg);
}
pos.func.dfg[inst].put_value_list(vlist);
}
fn legalized_type_for_sret(ty: Type) -> Type {
if ty.is_bool() {
let bits = std::cmp::max(8, ty.bits());
Type::int(bits).unwrap()
} else {
ty
}
}
fn legalize_type_for_sret_store(pos: &mut FuncCursor, val: Value, ty: Type) -> (Value, Type) {
if ty.is_bool() {
let bits = std::cmp::max(8, ty.bits());
let ty = Type::int(bits).unwrap();
let val = pos.ins().bint(ty, val);
(val, ty)
} else {
(val, ty)
}
}
pub fn handle_call_abi(
isa: &dyn TargetIsa,
mut inst: Inst,
func: &mut Function,
cfg: &ControlFlowGraph,
) -> bool {
let pos = &mut FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
let sig_ref = match check_call_signature(&pos.func.dfg, inst) {
Ok(_) => return spill_call_arguments(pos, isa),
Err(s) => s,
};
let sig = &pos.func.dfg.signatures[sig_ref];
let old_sig = &pos.func.dfg.old_signatures[sig_ref];
if sig.uses_struct_return_param()
&& old_sig
.as_ref()
.map_or(false, |s| !s.uses_struct_return_param())
{
legalize_sret_call(isa, pos, sig_ref, inst);
} else {
if !pos.func.dfg.signatures[sig_ref].returns.is_empty() {
inst = legalize_inst_results(pos, |func, abi_res| {
func.dfg.signatures[sig_ref].returns[abi_res]
});
}
}
pos.goto_inst(inst);
let abi_args = pos.func.dfg.signatures[sig_ref].params.len();
legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
func.dfg.signatures[sig_ref].params[abi_arg]
});
debug_assert!(
check_call_signature(&pos.func.dfg, inst).is_ok(),
"Signature still wrong: {}, {}{}",
pos.func.dfg.display_inst(inst, None),
sig_ref,
pos.func.dfg.signatures[sig_ref]
);
pos.goto_inst(inst);
spill_call_arguments(pos, isa);
true
}
pub fn handle_return_abi(inst: Inst, func: &mut Function, cfg: &ControlFlowGraph) -> bool {
if check_return_signature(&func.dfg, inst, &func.signature) {
return false;
}
let special_args = func
.signature
.returns
.iter()
.rev()
.take_while(|&rt| {
rt.purpose == ArgumentPurpose::Link
|| rt.purpose == ArgumentPurpose::StructReturn
|| rt.purpose == ArgumentPurpose::VMContext
})
.count();
let abi_args = func.signature.returns.len() - special_args;
let pos = &mut FuncCursor::new(func).at_inst(inst);
pos.use_srcloc(inst);
legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
let arg = func.signature.returns[abi_arg];
debug_assert!(
!arg.legalized_to_pointer,
"Return value cannot be legalized to pointer"
);
arg
});
if special_args > 0 {
debug!(
"Adding {} special-purpose arguments to {}",
special_args,
pos.func.dfg.display_inst(inst, None)
);
let mut vlist = pos.func.dfg[inst].take_value_list().unwrap();
let mut sret = None;
for arg in &pos.func.signature.returns[abi_args..] {
match arg.purpose {
ArgumentPurpose::Link
| ArgumentPurpose::StructReturn
| ArgumentPurpose::VMContext => {}
ArgumentPurpose::Normal => panic!("unexpected return value {}", arg),
_ => panic!("Unsupported special purpose return value {}", arg),
}
let idx = pos
.func
.signature
.params
.iter()
.rposition(|t| t.purpose == arg.purpose)
.expect("No matching special purpose argument.");
let val = pos
.func
.dfg
.block_params(pos.func.layout.entry_block().unwrap())[idx];
debug_assert_eq!(pos.func.dfg.value_type(val), arg.value_type);
vlist.push(val, &mut pos.func.dfg.value_lists);
if let ArgumentPurpose::StructReturn = arg.purpose {
sret = Some(val);
}
}
if let Some(sret) = sret {
let mut offset = 0;
let num_regular_rets = vlist.len(&pos.func.dfg.value_lists) - special_args;
for i in 0..num_regular_rets {
debug_assert_eq!(
pos.func.old_signature.as_ref().unwrap().returns[i].purpose,
ArgumentPurpose::Normal,
);
let v = vlist.get(0, &pos.func.dfg.value_lists).unwrap();
let ty = pos.func.dfg.value_type(v);
let (v, ty) = legalize_type_for_sret_store(pos, v, ty);
let size = ty.bytes();
offset = round_up_to_multiple_of_type_align(offset, ty);
pos.ins().store(MemFlags::trusted(), v, sret, offset as i32);
vlist.remove(0, &mut pos.func.dfg.value_lists);
offset += size;
}
}
pos.func.dfg[inst].put_value_list(vlist);
}
debug_assert_eq!(
pos.func.dfg.inst_variable_args(inst).len(),
abi_args + special_args
);
debug_assert!(
check_return_signature(&pos.func.dfg, inst, &pos.func.signature),
"Signature still wrong: {} / signature {}",
pos.func.dfg.display_inst(inst, None),
pos.func.signature
);
true
}
fn round_up_to_multiple_of_type_align(bytes: u32, ty: Type) -> u32 {
let align = ty.bytes();
round_up_to_multiple_of_pow2(bytes, align)
}
fn round_up_to_multiple_of_pow2(n: u32, to: u32) -> u32 {
debug_assert!(to > 0);
debug_assert!(to.is_power_of_two());
(n + to - 1) & !(to - 1)
}
fn spill_entry_params(func: &mut Function, entry: Block) {
for (abi, &arg) in func
.signature
.params
.iter()
.zip(func.dfg.block_params(entry))
{
if let ArgumentPurpose::StructArgument(_) = abi.purpose {
} else if let ArgumentLoc::Stack(offset) = abi.location {
let ss = func
.stack_slots
.make_incoming_arg(abi.value_type.bytes(), offset);
func.locations[arg] = ValueLoc::Stack(ss);
}
}
}
fn spill_call_arguments(pos: &mut FuncCursor, isa: &dyn TargetIsa) -> bool {
let inst = pos
.current_inst()
.expect("Cursor must point to a call instruction");
let sig_ref = pos
.func
.dfg
.call_signature(inst)
.expect("Call instruction expected.");
let arglist = {
let locations = &pos.func.locations;
let stack_slots = &mut pos.func.stack_slots;
pos.func
.dfg
.inst_variable_args(inst)
.iter()
.zip(&pos.func.dfg.signatures[sig_ref].params)
.enumerate()
.filter_map(|(idx, (&arg, abi))| {
match abi.location {
ArgumentLoc::Stack(offset) => {
let (ss, size) = match abi.purpose {
ArgumentPurpose::StructArgument(size) => {
(stack_slots.get_outgoing_arg(size, offset), Some(size))
}
_ => (
stack_slots.get_outgoing_arg(abi.value_type.bytes(), offset),
None,
),
};
if locations[arg] != ValueLoc::Stack(ss) {
Some((idx, arg, ss, size))
} else {
None
}
}
_ => None,
}
})
.collect::<Vec<_>>()
};
if arglist.is_empty() {
return false;
}
let mut libc_memcpy = None;
let mut import_memcpy = |func: &mut Function, pointer_type| {
if let Some(libc_memcpy) = libc_memcpy {
return libc_memcpy;
}
let signature = {
let mut s = Signature::new(isa.default_call_conv());
s.params.push(AbiParam::new(pointer_type));
s.params.push(AbiParam::new(pointer_type));
s.params.push(AbiParam::new(pointer_type));
legalize_libcall_signature(&mut s, isa);
func.import_signature(s)
};
let func = func.import_function(ExtFuncData {
name: ExternalName::LibCall(LibCall::Memcpy),
signature,
colocated: false,
});
libc_memcpy = Some(func);
func
};
for (idx, arg, ss, size) in arglist {
let stack_val = if let Some(size) = size {
let pointer_type = pos.func.dfg.value_type(arg);
let src = arg;
let dest = pos.ins().stack_addr(pointer_type, ss, 0);
let size = pos.ins().iconst(pointer_type, i64::from(size));
let libc_memcpy = import_memcpy(pos.func, pointer_type);
pos.ins().call(libc_memcpy, &[dest, src, size]);
pos.ins().dummy_sarg_t()
} else {
pos.ins().spill(arg)
};
pos.func.locations[stack_val] = ValueLoc::Stack(ss);
pos.func.dfg.inst_variable_args_mut(inst)[idx] = stack_val;
}
true
}
#[cfg(test)]
mod tests {
use super::round_up_to_multiple_of_pow2;
#[test]
fn round_up_to_multiple_of_pow2_works() {
for (n, to, expected) in vec![
(0, 1, 0),
(1, 1, 1),
(2, 1, 2),
(0, 2, 0),
(1, 2, 2),
(2, 2, 2),
(3, 2, 4),
(0, 4, 0),
(1, 4, 4),
(2, 4, 4),
(3, 4, 4),
(4, 4, 4),
(5, 4, 8),
] {
let actual = round_up_to_multiple_of_pow2(n, to);
assert_eq!(
actual, expected,
"round_up_to_multiple_of_pow2(n = {}, to = {}) = {} (expected {})",
n, to, actual, expected
);
}
}
}