use super::super::settings as shared_settings;
use super::registers::{FPR, GPR, RU};
use super::settings as isa_settings;
use crate::abi::{legalize_args, ArgAction, ArgAssigner, ValueConversion};
use crate::cursor::{Cursor, CursorPosition, EncCursor};
use crate::ir;
use crate::ir::immediates::Imm64;
use crate::ir::stackslot::{StackOffset, StackSize};
use crate::ir::types;
use crate::ir::{
get_probestack_funcref, AbiParam, ArgumentExtension, ArgumentLoc, ArgumentPurpose, InstBuilder,
ValueLoc,
};
use crate::isa::{CallConv, RegClass, RegUnit, TargetIsa};
use crate::regalloc::RegisterSet;
use crate::result::CodegenResult;
use crate::stack_layout::layout_stack;
use alloc::borrow::Cow;
use core::i32;
use target_lexicon::{PointerWidth, Triple};
static ARG_GPRS: [RU; 6] = [RU::rdi, RU::rsi, RU::rdx, RU::rcx, RU::r8, RU::r9];
static RET_GPRS: [RU; 3] = [RU::rax, RU::rdx, RU::rcx];
static ARG_GPRS_WIN_FASTCALL_X64: [RU; 4] = [RU::rcx, RU::rdx, RU::r8, RU::r9];
static RET_GPRS_WIN_FASTCALL_X64: [RU; 1] = [RU::rax];
const WIN_SHADOW_STACK_SPACE: StackSize = 32;
const STACK_ALIGNMENT: u32 = 16;
#[derive(Clone)]
struct Args {
pointer_bytes: u8,
pointer_bits: u8,
pointer_type: ir::Type,
gpr: &'static [RU],
gpr_used: usize,
fpr_limit: usize,
fpr_used: usize,
offset: u32,
call_conv: CallConv,
shared_flags: shared_settings::Flags,
#[allow(dead_code)]
isa_flags: isa_settings::Flags,
assigning_returns: bool,
}
impl Args {
fn new(
bits: u8,
gpr: &'static [RU],
fpr_limit: usize,
call_conv: CallConv,
shared_flags: &shared_settings::Flags,
isa_flags: &isa_settings::Flags,
assigning_returns: bool,
) -> Self {
let offset = if call_conv.extends_windows_fastcall() {
WIN_SHADOW_STACK_SPACE
} else {
0
};
Self {
pointer_bytes: bits / 8,
pointer_bits: bits,
pointer_type: ir::Type::int(u16::from(bits)).unwrap(),
gpr,
gpr_used: 0,
fpr_limit,
fpr_used: 0,
offset,
call_conv,
shared_flags: shared_flags.clone(),
isa_flags: isa_flags.clone(),
assigning_returns,
}
}
}
impl ArgAssigner for Args {
fn assign(&mut self, arg: &AbiParam) -> ArgAction {
if let ArgumentPurpose::StructArgument(size) = arg.purpose {
if self.call_conv != CallConv::SystemV {
panic!(
"The sarg argument purpose is not yet implemented for non-systemv call conv {:?}",
self.call_conv,
);
}
let loc = ArgumentLoc::Stack(self.offset as i32);
self.offset += size;
debug_assert!(self.offset <= i32::MAX as u32);
return ArgAction::AssignAndChangeType(loc, types::SARG_T);
}
let ty = arg.value_type;
if ty.bits() > u16::from(self.pointer_bits) {
if !self.assigning_returns && self.call_conv.extends_windows_fastcall() {
return ValueConversion::Pointer(self.pointer_type).into();
} else if !ty.is_vector() && !ty.is_float() {
return ValueConversion::IntSplit.into();
}
}
if ty.is_vector() {
if self.shared_flags.enable_simd() {
let reg = FPR.unit(self.fpr_used);
self.fpr_used += 1;
return ArgumentLoc::Reg(reg).into();
}
return ValueConversion::VectorSplit.into();
}
if ty.is_int()
&& ty.bits() < u16::from(self.pointer_bits)
&& self.call_conv.extends_baldrdash()
{
match arg.extension {
ArgumentExtension::None => {}
ArgumentExtension::Uext => return ValueConversion::Uext(self.pointer_type).into(),
ArgumentExtension::Sext => return ValueConversion::Sext(self.pointer_type).into(),
}
}
if ty.is_int() && self.call_conv.extends_baldrdash() {
match arg.purpose {
ArgumentPurpose::VMContext => {
return ArgumentLoc::Reg(if self.pointer_bits == 64 {
RU::r14
} else {
RU::rsi
} as RegUnit)
.into();
}
ArgumentPurpose::SignatureId => {
return ArgumentLoc::Reg(if self.pointer_bits == 64 {
RU::r10
} else {
RU::rcx
} as RegUnit)
.into()
}
_ => {}
}
}
if !ty.is_float() && self.gpr_used < self.gpr.len() {
let reg = self.gpr[self.gpr_used] as RegUnit;
self.gpr_used += 1;
return ArgumentLoc::Reg(reg).into();
}
let fpr_offset = if self.call_conv.extends_windows_fastcall() {
debug_assert_eq!(self.fpr_limit, self.gpr.len());
&mut self.gpr_used
} else {
&mut self.fpr_used
};
if ty.is_float() && *fpr_offset < self.fpr_limit {
let reg = FPR.unit(*fpr_offset);
*fpr_offset += 1;
return ArgumentLoc::Reg(reg).into();
}
let loc = ArgumentLoc::Stack(self.offset as i32);
self.offset += u32::from(self.pointer_bytes);
debug_assert!(self.offset <= i32::MAX as u32);
loc.into()
}
}
pub fn legalize_signature(
sig: &mut Cow<ir::Signature>,
triple: &Triple,
_current: bool,
shared_flags: &shared_settings::Flags,
isa_flags: &isa_settings::Flags,
) {
let bits;
let mut args;
match triple.pointer_width().unwrap() {
PointerWidth::U16 => panic!(),
PointerWidth::U32 => {
bits = 32;
args = Args::new(bits, &[], 0, sig.call_conv, shared_flags, isa_flags, false);
}
PointerWidth::U64 => {
bits = 64;
args = if sig.call_conv.extends_windows_fastcall() {
Args::new(
bits,
&ARG_GPRS_WIN_FASTCALL_X64[..],
4,
sig.call_conv,
shared_flags,
isa_flags,
false,
)
} else {
Args::new(
bits,
&ARG_GPRS[..],
8,
sig.call_conv,
shared_flags,
isa_flags,
false,
)
};
}
}
let (ret_regs, ret_fpr_limit) = if sig.call_conv.extends_windows_fastcall() {
(&RET_GPRS_WIN_FASTCALL_X64[..], 1)
} else {
(&RET_GPRS[..], 2)
};
let mut rets = Args::new(
bits,
ret_regs,
ret_fpr_limit,
sig.call_conv,
shared_flags,
isa_flags,
true,
);
let mut backup_rets = rets.clone();
if let Some(new_returns) = legalize_args(&sig.returns, &mut rets) {
if new_returns
.iter()
.filter(|r| r.purpose == ArgumentPurpose::Normal)
.any(|r| !r.location.is_reg())
{
debug_assert!(!sig.uses_struct_return_param());
let mut ret_ptr_param = AbiParam {
value_type: args.pointer_type,
purpose: ArgumentPurpose::StructReturn,
extension: ArgumentExtension::None,
location: ArgumentLoc::Unassigned,
legalized_to_pointer: false,
};
match args.assign(&ret_ptr_param) {
ArgAction::Assign(ArgumentLoc::Reg(reg)) => {
ret_ptr_param.location = ArgumentLoc::Reg(reg);
sig.to_mut().params.push(ret_ptr_param);
}
_ => unreachable!("return pointer should always get a register assignment"),
}
let mut ret_ptr_return = AbiParam {
value_type: args.pointer_type,
purpose: ArgumentPurpose::StructReturn,
extension: ArgumentExtension::None,
location: ArgumentLoc::Unassigned,
legalized_to_pointer: false,
};
match backup_rets.assign(&ret_ptr_return) {
ArgAction::Assign(ArgumentLoc::Reg(reg)) => {
ret_ptr_return.location = ArgumentLoc::Reg(reg);
sig.to_mut().returns.push(ret_ptr_return);
}
_ => unreachable!("return pointer should always get a register assignment"),
}
sig.to_mut().returns.retain(|ret| {
debug_assert_eq!(
ret.location.is_assigned(),
ret.purpose != ArgumentPurpose::Normal
);
ret.location.is_assigned()
});
if let Some(new_returns) = legalize_args(&sig.returns, &mut backup_rets) {
sig.to_mut().returns = new_returns;
}
} else {
sig.to_mut().returns = new_returns;
}
}
if let Some(new_params) = legalize_args(&sig.params, &mut args) {
sig.to_mut().params = new_params;
}
}
pub fn regclass_for_abi_type(ty: ir::Type) -> RegClass {
if ty.is_int() || ty.is_bool() || ty.is_ref() {
GPR
} else {
FPR
}
}
pub fn allocatable_registers(triple: &Triple, flags: &shared_settings::Flags) -> RegisterSet {
let mut regs = RegisterSet::new();
regs.take(GPR, RU::rsp as RegUnit);
regs.take(GPR, RU::rbp as RegUnit);
if triple.pointer_width().unwrap() != PointerWidth::U64 {
for i in 8..16 {
regs.take(GPR, GPR.unit(i));
regs.take(FPR, FPR.unit(i));
}
if flags.enable_pinned_reg() {
unimplemented!("Pinned register not implemented on x86-32.");
}
} else {
if flags.enable_pinned_reg() {
regs.take(GPR, RU::r15 as RegUnit);
}
}
regs
}
fn callee_saved_gprs(isa: &dyn TargetIsa, call_conv: CallConv) -> &'static [RU] {
match isa.triple().pointer_width().unwrap() {
PointerWidth::U16 => panic!(),
PointerWidth::U32 => &[RU::rbx, RU::rsi, RU::rdi],
PointerWidth::U64 => {
if call_conv.extends_windows_fastcall() {
&[
RU::rbx,
RU::rdi,
RU::rsi,
RU::r12,
RU::r13,
RU::r14,
RU::r15,
]
} else {
&[RU::rbx, RU::r12, RU::r13, RU::r14, RU::r15]
}
}
}
}
fn callee_saved_fprs(isa: &dyn TargetIsa, call_conv: CallConv) -> &'static [RU] {
match isa.triple().pointer_width().unwrap() {
PointerWidth::U16 => panic!(),
PointerWidth::U32 => &[],
PointerWidth::U64 => {
if call_conv.extends_windows_fastcall() {
&[
RU::xmm6,
RU::xmm7,
RU::xmm8,
RU::xmm9,
RU::xmm10,
RU::xmm11,
RU::xmm12,
RU::xmm13,
RU::xmm14,
RU::xmm15,
]
} else {
&[]
}
}
}
}
fn callee_saved_regs_used(isa: &dyn TargetIsa, func: &ir::Function) -> RegisterSet {
let mut all_callee_saved = RegisterSet::empty();
for reg in callee_saved_gprs(isa, func.signature.call_conv) {
all_callee_saved.free(GPR, *reg as RegUnit);
}
for reg in callee_saved_fprs(isa, func.signature.call_conv) {
all_callee_saved.free(FPR, *reg as RegUnit);
}
let mut used = RegisterSet::empty();
for value_loc in func.locations.values() {
if let ValueLoc::Reg(ru) = *value_loc {
if GPR.contains(ru) {
if !used.is_avail(GPR, ru) {
used.free(GPR, ru);
}
} else if FPR.contains(ru) {
if !used.is_avail(FPR, ru) {
used.free(FPR, ru);
}
}
}
}
for block in &func.layout {
for inst in func.layout.block_insts(block) {
match func.dfg[inst] {
ir::instructions::InstructionData::RegMove { dst, .. }
| ir::instructions::InstructionData::RegFill { dst, .. } => {
if GPR.contains(dst) {
if !used.is_avail(GPR, dst) {
used.free(GPR, dst);
}
} else if FPR.contains(dst) {
if !used.is_avail(FPR, dst) {
used.free(FPR, dst);
}
}
}
_ => (),
}
}
}
used.intersect(&all_callee_saved);
used
}
pub fn prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> CodegenResult<()> {
match func.signature.call_conv {
CallConv::Fast | CallConv::Cold | CallConv::SystemV => {
system_v_prologue_epilogue(func, isa)
}
CallConv::WindowsFastcall => fastcall_prologue_epilogue(func, isa),
CallConv::BaldrdashSystemV | CallConv::BaldrdashWindows => {
baldrdash_prologue_epilogue(func, isa)
}
CallConv::Probestack => unimplemented!("probestack calling convention"),
CallConv::Baldrdash2020 => unimplemented!("Baldrdash ABI 2020"),
}
}
fn baldrdash_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> CodegenResult<()> {
debug_assert!(
!isa.flags().enable_probestack(),
"baldrdash does not expect cranelift to emit stack probes"
);
let word_size = StackSize::from(isa.pointer_bytes());
let shadow_store_size = if func.signature.call_conv.extends_windows_fastcall() {
WIN_SHADOW_STACK_SPACE
} else {
0
};
let bytes =
StackSize::from(isa.flags().baldrdash_prologue_words()) * word_size + shadow_store_size;
let mut ss = ir::StackSlotData::new(ir::StackSlotKind::IncomingArg, bytes);
ss.offset = Some(-(bytes as StackOffset));
func.stack_slots.push(ss);
let is_leaf = func.is_leaf();
layout_stack(&mut func.stack_slots, is_leaf, STACK_ALIGNMENT)?;
Ok(())
}
fn fastcall_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> CodegenResult<()> {
if isa.triple().pointer_width().unwrap() != PointerWidth::U64 {
panic!("TODO: windows-fastcall: x86-32 not implemented yet");
}
let csrs = callee_saved_regs_used(isa, func);
let gpsr_stack_size = ((csrs.iter(GPR).len() + 2) * isa.pointer_bytes() as usize) as u32;
let fpsr_stack_size = (csrs.iter(FPR).len() * types::F64X2.bytes() as usize) as u32;
let mut csr_stack_size = gpsr_stack_size + fpsr_stack_size;
if fpsr_stack_size > 0 {
csr_stack_size = (csr_stack_size + 15) & !15;
}
func.create_stack_slot(ir::StackSlotData {
kind: ir::StackSlotKind::IncomingArg,
size: csr_stack_size,
offset: Some(-(csr_stack_size as StackOffset)),
});
let is_leaf = func.is_leaf();
if !is_leaf {
func.create_stack_slot(ir::StackSlotData {
kind: ir::StackSlotKind::ExplicitSlot,
size: WIN_SHADOW_STACK_SPACE,
offset: None,
});
}
let total_stack_size = layout_stack(&mut func.stack_slots, is_leaf, STACK_ALIGNMENT)? as i32;
let local_stack_size = i64::from(total_stack_size - gpsr_stack_size as i32);
let reg_type = isa.pointer_type();
let sp_arg_index = if fpsr_stack_size > 0 {
let sp_arg = ir::AbiParam::special_reg(
reg_type,
ir::ArgumentPurpose::CalleeSaved,
RU::rsp as RegUnit,
);
let index = func.signature.params.len();
func.signature.params.push(sp_arg);
Some(index)
} else {
None
};
let fp_arg = ir::AbiParam::special_reg(
reg_type,
ir::ArgumentPurpose::FramePointer,
RU::rbp as RegUnit,
);
func.signature.params.push(fp_arg);
func.signature.returns.push(fp_arg);
for gp_csr in csrs.iter(GPR) {
let csr_arg = ir::AbiParam::special_reg(reg_type, ir::ArgumentPurpose::CalleeSaved, gp_csr);
func.signature.params.push(csr_arg);
func.signature.returns.push(csr_arg);
}
for fp_csr in csrs.iter(FPR) {
let csr_arg =
ir::AbiParam::special_reg(types::F64X2, ir::ArgumentPurpose::CalleeSaved, fp_csr);
func.signature.params.push(csr_arg);
func.signature.returns.push(csr_arg);
}
let entry_block = func.layout.entry_block().expect("missing entry block");
let mut pos = EncCursor::new(func, isa).at_first_insertion_point(entry_block);
insert_common_prologue(
&mut pos,
local_stack_size,
reg_type,
&csrs,
sp_arg_index.is_some(),
isa,
);
let mut pos = pos.at_position(CursorPosition::Nowhere);
insert_common_epilogues(&mut pos, local_stack_size, reg_type, &csrs, sp_arg_index);
Ok(())
}
fn system_v_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> CodegenResult<()> {
let pointer_width = isa.triple().pointer_width().unwrap();
let word_size = pointer_width.bytes() as usize;
let csrs = callee_saved_regs_used(isa, func);
assert!(
csrs.iter(FPR).len() == 0,
"SysV ABI does not have callee-save SIMD registers"
);
let csr_stack_size = ((csrs.iter(GPR).len() + 2) * word_size) as i32;
func.create_stack_slot(ir::StackSlotData {
kind: ir::StackSlotKind::IncomingArg,
size: csr_stack_size as u32,
offset: Some(-csr_stack_size),
});
let is_leaf = func.is_leaf();
let total_stack_size = layout_stack(&mut func.stack_slots, is_leaf, STACK_ALIGNMENT)? as i32;
let local_stack_size = i64::from(total_stack_size - csr_stack_size);
let reg_type = ir::Type::int(u16::from(pointer_width.bits())).unwrap();
let sp_arg_index = if isa.pointer_bits() == 32 {
let sp_arg = ir::AbiParam::special_reg(
reg_type,
ir::ArgumentPurpose::CalleeSaved,
RU::rsp as RegUnit,
);
let index = func.signature.params.len();
func.signature.params.push(sp_arg);
Some(index)
} else {
None
};
let fp_arg = ir::AbiParam::special_reg(
reg_type,
ir::ArgumentPurpose::FramePointer,
RU::rbp as RegUnit,
);
func.signature.params.push(fp_arg);
func.signature.returns.push(fp_arg);
for csr in csrs.iter(GPR) {
let csr_arg = ir::AbiParam::special_reg(reg_type, ir::ArgumentPurpose::CalleeSaved, csr);
func.signature.params.push(csr_arg);
func.signature.returns.push(csr_arg);
}
let entry_block = func.layout.entry_block().expect("missing entry block");
let mut pos = EncCursor::new(func, isa).at_first_insertion_point(entry_block);
insert_common_prologue(
&mut pos,
local_stack_size,
reg_type,
&csrs,
sp_arg_index.is_some(),
isa,
);
let mut pos = pos.at_position(CursorPosition::Nowhere);
insert_common_epilogues(&mut pos, local_stack_size, reg_type, &csrs, sp_arg_index);
Ok(())
}
fn insert_common_prologue(
pos: &mut EncCursor,
stack_size: i64,
reg_type: ir::types::Type,
csrs: &RegisterSet,
has_sp_param: bool,
isa: &dyn TargetIsa,
) {
let sp = if has_sp_param {
let block = pos.current_block().expect("missing block under cursor");
let sp = pos.func.dfg.append_block_param(block, reg_type);
pos.func.locations[sp] = ir::ValueLoc::Reg(RU::rsp as RegUnit);
Some(sp)
} else {
None
};
if stack_size > 0 || !pos.func.is_leaf() {
let scratch = ir::ValueLoc::Reg(RU::rax as RegUnit);
let stack_limit_arg = match pos.func.special_param(ArgumentPurpose::StackLimit) {
Some(arg) => {
let copy = pos.ins().copy(arg);
pos.func.locations[copy] = scratch;
Some(copy)
}
None => pos
.func
.stack_limit
.map(|gv| interpret_gv(pos, gv, sp, scratch)),
};
if let Some(stack_limit_arg) = stack_limit_arg {
insert_stack_check(pos, stack_size, stack_limit_arg);
}
}
let block = pos.current_block().expect("missing block under cursor");
let fp = pos.func.dfg.append_block_param(block, reg_type);
pos.func.locations[fp] = ir::ValueLoc::Reg(RU::rbp as RegUnit);
pos.ins().x86_push(fp);
let mov_sp_inst = pos
.ins()
.copy_special(RU::rsp as RegUnit, RU::rbp as RegUnit);
let mut last_csr_push = None;
for reg in csrs.iter(GPR) {
let csr_arg = pos.func.dfg.append_block_param(block, reg_type);
pos.func.locations[csr_arg] = ir::ValueLoc::Reg(reg);
last_csr_push = Some(pos.ins().x86_push(csr_arg));
}
let mut adjust_sp_inst = None;
if stack_size > 0 {
if isa.flags().enable_probestack() && stack_size > (1 << isa.flags().probestack_size_log2())
{
let rax = RU::rax as RegUnit;
let rax_val = ir::ValueLoc::Reg(rax);
let arg = pos.ins().iconst(reg_type, stack_size);
pos.func.locations[arg] = rax_val;
let callee = get_probestack_funcref(pos.func, reg_type, rax, isa);
let call = if !isa.flags().is_pic()
&& isa.triple().pointer_width().unwrap() == PointerWidth::U64
&& !pos.func.dfg.ext_funcs[callee].colocated
{
let r11 = RU::r11 as RegUnit;
let sig = pos.func.dfg.ext_funcs[callee].signature;
let addr = pos.ins().func_addr(reg_type, callee);
pos.func.locations[addr] = ir::ValueLoc::Reg(r11);
pos.ins().call_indirect(sig, addr, &[arg])
} else {
pos.ins().call(callee, &[arg])
};
if !isa.flags().probestack_func_adjusts_sp() {
let result = pos.func.dfg.inst_results(call)[0];
pos.func.locations[result] = rax_val;
adjust_sp_inst = Some(pos.ins().adjust_sp_down(result));
}
} else {
adjust_sp_inst = Some(pos.ins().adjust_sp_down_imm(Imm64::new(stack_size)));
}
}
let mut last_fpr_save = None;
for (i, reg) in csrs.iter(FPR).enumerate() {
let csr_arg = pos.func.dfg.append_block_param(block, types::F64X2);
pos.func.locations[csr_arg] = ir::ValueLoc::Reg(reg);
let offset = ((i + 1) * types::F64X2.bytes() as usize) as i64
+ (stack_size % types::F64X2.bytes() as i64);
last_fpr_save = Some(pos.ins().store(
ir::MemFlags::trusted(),
csr_arg,
sp.expect("FPR save requires SP param"),
(stack_size - offset) as i32,
));
}
pos.func.prologue_end = Some(
last_fpr_save
.or(adjust_sp_inst)
.or(last_csr_push)
.unwrap_or(mov_sp_inst),
);
}
fn interpret_gv(
pos: &mut EncCursor,
gv: ir::GlobalValue,
sp: Option<ir::Value>,
scratch: ir::ValueLoc,
) -> ir::Value {
match pos.func.global_values[gv] {
ir::GlobalValueData::VMContext => {
let vmctx_index = pos
.func
.signature
.special_param_index(ir::ArgumentPurpose::VMContext)
.expect("no vmcontext parameter found");
match pos.func.signature.params[vmctx_index] {
AbiParam {
location: ArgumentLoc::Reg(_),
..
} => {
let entry = pos.func.layout.entry_block().unwrap();
pos.func.dfg.block_params(entry)[vmctx_index]
}
AbiParam {
location: ArgumentLoc::Stack(offset),
value_type,
..
} => {
let offset =
offset + i32::from(pos.isa.pointer_bytes() * (1 + vmctx_index as u8));
let ret =
pos.ins()
.load(value_type, ir::MemFlags::trusted(), sp.unwrap(), offset);
pos.func.locations[ret] = scratch;
return ret;
}
AbiParam {
location: ArgumentLoc::Unassigned,
..
} => unreachable!(),
}
}
ir::GlobalValueData::Load {
base,
offset,
global_type,
readonly: _,
} => {
let base = interpret_gv(pos, base, sp, scratch);
let ret = pos
.ins()
.load(global_type, ir::MemFlags::trusted(), base, offset);
pos.func.locations[ret] = scratch;
return ret;
}
ref other => panic!("global value for stack limit not supported: {}", other),
}
}
fn insert_stack_check(pos: &mut EncCursor, stack_size: i64, stack_limit_arg: ir::Value) {
use crate::ir::condcodes::IntCC;
if stack_size >= 32 * 1024 {
let cflags = pos.ins().ifcmp_sp(stack_limit_arg);
pos.func.locations[cflags] = ir::ValueLoc::Reg(RU::rflags as RegUnit);
pos.ins().trapif(
IntCC::UnsignedGreaterThanOrEqual,
cflags,
ir::TrapCode::StackOverflow,
);
}
let sp_threshold = pos.ins().iadd_imm(stack_limit_arg, stack_size);
pos.func.locations[sp_threshold] = ir::ValueLoc::Reg(RU::rax as RegUnit);
let cflags = pos.ins().ifcmp_sp(sp_threshold);
pos.func.locations[cflags] = ir::ValueLoc::Reg(RU::rflags as RegUnit);
pos.ins().trapif(
IntCC::UnsignedGreaterThanOrEqual,
cflags,
ir::TrapCode::StackOverflow,
);
}
fn insert_common_epilogues(
pos: &mut EncCursor,
stack_size: i64,
reg_type: ir::types::Type,
csrs: &RegisterSet,
sp_arg_index: Option<usize>,
) {
while let Some(block) = pos.next_block() {
pos.goto_last_inst(block);
if let Some(inst) = pos.current_inst() {
if pos.func.dfg[inst].opcode().is_return() {
insert_common_epilogue(inst, block, stack_size, pos, reg_type, csrs, sp_arg_index);
}
}
}
}
fn insert_common_epilogue(
inst: ir::Inst,
block: ir::Block,
stack_size: i64,
pos: &mut EncCursor,
reg_type: ir::types::Type,
csrs: &RegisterSet,
sp_arg_index: Option<usize>,
) {
let fp_pop = pos.ins().x86_pop(reg_type);
let fp_pop_inst = pos.prev_inst().unwrap();
pos.func.locations[fp_pop] = ir::ValueLoc::Reg(RU::rbp as RegUnit);
pos.func.dfg.append_inst_arg(inst, fp_pop);
let mut first_csr_pop_inst = None;
for reg in csrs.iter(GPR) {
let csr_pop = pos.ins().x86_pop(reg_type);
first_csr_pop_inst = pos.prev_inst();
assert!(first_csr_pop_inst.is_some());
pos.func.locations[csr_pop] = ir::ValueLoc::Reg(reg);
pos.func.dfg.append_inst_arg(inst, csr_pop);
}
let mut sp_adjust_inst = None;
if stack_size > 0 {
pos.ins().adjust_sp_up_imm(Imm64::new(stack_size));
sp_adjust_inst = pos.prev_inst();
assert!(sp_adjust_inst.is_some());
}
let mut first_fpr_load = None;
if let Some(index) = sp_arg_index {
let sp = pos
.func
.dfg
.block_params(pos.func.layout.entry_block().unwrap())[index];
for (i, reg) in csrs.iter(FPR).enumerate() {
let offset = ((i + 1) * types::F64X2.bytes() as usize) as i64
+ (stack_size % types::F64X2.bytes() as i64);
let value = pos.ins().load(
types::F64X2,
ir::MemFlags::trusted(),
sp,
(stack_size - offset) as i32,
);
first_fpr_load.get_or_insert(pos.current_inst().expect("current inst"));
pos.func.locations[value] = ir::ValueLoc::Reg(reg);
pos.func.dfg.append_inst_arg(inst, value);
}
} else {
assert!(csrs.iter(FPR).len() == 0);
}
pos.func.epilogues_start.push((
first_fpr_load
.or(sp_adjust_inst)
.or(first_csr_pop_inst)
.unwrap_or(fp_pop_inst),
block,
));
}
#[cfg(feature = "unwind")]
pub fn create_unwind_info(
func: &ir::Function,
isa: &dyn TargetIsa,
) -> CodegenResult<Option<crate::isa::unwind::UnwindInfo>> {
use crate::isa::unwind::UnwindInfo;
Ok(match func.signature.call_conv {
CallConv::Fast | CallConv::Cold | CallConv::SystemV => {
super::unwind::systemv::create_unwind_info(func, isa)?.map(|u| UnwindInfo::SystemV(u))
}
CallConv::WindowsFastcall => {
super::unwind::winx64::create_unwind_info(func, isa)?.map(|u| UnwindInfo::WindowsX64(u))
}
_ => None,
})
}