9cba14b95b
the behavior of the type system not only depends on the current assumptions, but also the currentnphase of the compiler. This is mostly necessary as we need to decide whether and how to reveal opaque types. We track this via the `TypingMode`.
929 lines
36 KiB
Rust
929 lines
36 KiB
Rust
//! Codegen of `asm!` invocations.
|
|
|
|
use std::fmt::Write;
|
|
|
|
use cranelift_codegen::isa::CallConv;
|
|
use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
|
|
use rustc_hir::LangItem;
|
|
use rustc_span::sym;
|
|
use rustc_target::asm::*;
|
|
use target_lexicon::BinaryFormat;
|
|
|
|
use crate::prelude::*;
|
|
|
|
pub(crate) enum CInlineAsmOperand<'tcx> {
|
|
In {
|
|
reg: InlineAsmRegOrRegClass,
|
|
value: Value,
|
|
},
|
|
Out {
|
|
reg: InlineAsmRegOrRegClass,
|
|
late: bool,
|
|
place: Option<CPlace<'tcx>>,
|
|
},
|
|
InOut {
|
|
reg: InlineAsmRegOrRegClass,
|
|
_late: bool,
|
|
in_value: Value,
|
|
out_place: Option<CPlace<'tcx>>,
|
|
},
|
|
Const {
|
|
value: String,
|
|
},
|
|
Symbol {
|
|
symbol: String,
|
|
},
|
|
}
|
|
|
|
pub(crate) fn codegen_inline_asm_terminator<'tcx>(
|
|
fx: &mut FunctionCx<'_, '_, 'tcx>,
|
|
span: Span,
|
|
template: &[InlineAsmTemplatePiece],
|
|
operands: &[InlineAsmOperand<'tcx>],
|
|
options: InlineAsmOptions,
|
|
destination: Option<mir::BasicBlock>,
|
|
) {
|
|
// Used by panic_abort on Windows, but uses a syntax which only happens to work with
|
|
// asm!() by accident and breaks with the GNU assembler as well as global_asm!() for
|
|
// the LLVM backend.
|
|
if template.len() == 1 && template[0] == InlineAsmTemplatePiece::String("int $$0x29".into()) {
|
|
fx.bcx.ins().trap(TrapCode::user(2).unwrap());
|
|
return;
|
|
}
|
|
|
|
let operands = operands
|
|
.iter()
|
|
.map(|operand| match *operand {
|
|
InlineAsmOperand::In { reg, ref value } => CInlineAsmOperand::In {
|
|
reg,
|
|
value: crate::base::codegen_operand(fx, value).load_scalar(fx),
|
|
},
|
|
InlineAsmOperand::Out { reg, late, ref place } => CInlineAsmOperand::Out {
|
|
reg,
|
|
late,
|
|
place: place.map(|place| crate::base::codegen_place(fx, place)),
|
|
},
|
|
InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => {
|
|
CInlineAsmOperand::InOut {
|
|
reg,
|
|
_late: late,
|
|
in_value: crate::base::codegen_operand(fx, in_value).load_scalar(fx),
|
|
out_place: out_place.map(|place| crate::base::codegen_place(fx, place)),
|
|
}
|
|
}
|
|
InlineAsmOperand::Const { ref value } => {
|
|
let (const_value, ty) = crate::constant::eval_mir_constant(fx, value);
|
|
let value = rustc_codegen_ssa::common::asm_const_to_str(
|
|
fx.tcx,
|
|
span,
|
|
const_value,
|
|
fx.layout_of(ty),
|
|
);
|
|
CInlineAsmOperand::Const { value }
|
|
}
|
|
InlineAsmOperand::SymFn { ref value } => {
|
|
if cfg!(not(feature = "inline_asm_sym")) {
|
|
fx.tcx
|
|
.dcx()
|
|
.span_err(span, "asm! and global_asm! sym operands are not yet supported");
|
|
}
|
|
|
|
let const_ = fx.monomorphize(value.const_);
|
|
if let ty::FnDef(def_id, args) = *const_.ty().kind() {
|
|
let instance = ty::Instance::resolve_for_fn_ptr(
|
|
fx.tcx,
|
|
ty::TypingEnv::fully_monomorphized(),
|
|
def_id,
|
|
args,
|
|
)
|
|
.unwrap();
|
|
let symbol = fx.tcx.symbol_name(instance);
|
|
|
|
// Pass a wrapper rather than the function itself as the function itself may not
|
|
// be exported from the main codegen unit and may thus be unreachable from the
|
|
// object file created by an external assembler.
|
|
let inline_asm_index = fx.cx.inline_asm_index.get();
|
|
fx.cx.inline_asm_index.set(inline_asm_index + 1);
|
|
let wrapper_name = format!(
|
|
"__inline_asm_{}_wrapper_n{}",
|
|
fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
|
|
inline_asm_index
|
|
);
|
|
let sig =
|
|
get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance);
|
|
create_wrapper_function(fx.module, sig, &wrapper_name, symbol.name);
|
|
|
|
CInlineAsmOperand::Symbol { symbol: wrapper_name }
|
|
} else {
|
|
span_bug!(span, "invalid type for asm sym (fn)");
|
|
}
|
|
}
|
|
InlineAsmOperand::SymStatic { def_id } => {
|
|
assert!(fx.tcx.is_static(def_id));
|
|
let instance = Instance::mono(fx.tcx, def_id);
|
|
CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
|
|
}
|
|
InlineAsmOperand::Label { .. } => {
|
|
span_bug!(span, "asm! label operands are not yet supported");
|
|
}
|
|
})
|
|
.collect::<Vec<_>>();
|
|
|
|
codegen_inline_asm_inner(fx, template, &operands, options);
|
|
|
|
match destination {
|
|
Some(destination) => {
|
|
let destination_block = fx.get_block(destination);
|
|
fx.bcx.ins().jump(destination_block, &[]);
|
|
}
|
|
None => {
|
|
fx.bcx.ins().trap(TrapCode::user(0 /* unreachable */).unwrap());
|
|
}
|
|
}
|
|
}
|
|
|
|
pub(crate) fn codegen_inline_asm_inner<'tcx>(
|
|
fx: &mut FunctionCx<'_, '_, 'tcx>,
|
|
template: &[InlineAsmTemplatePiece],
|
|
operands: &[CInlineAsmOperand<'tcx>],
|
|
options: InlineAsmOptions,
|
|
) {
|
|
// FIXME add .eh_frame unwind info directives
|
|
|
|
let mut asm_gen = InlineAssemblyGenerator {
|
|
tcx: fx.tcx,
|
|
arch: fx.tcx.sess.asm_arch.unwrap(),
|
|
enclosing_def_id: fx.instance.def_id(),
|
|
template,
|
|
operands,
|
|
options,
|
|
registers: Vec::new(),
|
|
stack_slots_clobber: Vec::new(),
|
|
stack_slots_input: Vec::new(),
|
|
stack_slots_output: Vec::new(),
|
|
stack_slot_size: Size::from_bytes(0),
|
|
is_naked: false,
|
|
};
|
|
asm_gen.allocate_registers();
|
|
asm_gen.allocate_stack_slots();
|
|
|
|
let inline_asm_index = fx.cx.inline_asm_index.get();
|
|
fx.cx.inline_asm_index.set(inline_asm_index + 1);
|
|
let asm_name = format!(
|
|
"__inline_asm_{}_n{}",
|
|
fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
|
|
inline_asm_index
|
|
);
|
|
|
|
let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
|
|
fx.cx.global_asm.push_str(&generated_asm);
|
|
|
|
let mut inputs = Vec::new();
|
|
let mut outputs = Vec::new();
|
|
for (i, operand) in operands.iter().enumerate() {
|
|
match operand {
|
|
CInlineAsmOperand::In { reg: _, value } => {
|
|
inputs.push((asm_gen.stack_slots_input[i].unwrap(), *value));
|
|
}
|
|
CInlineAsmOperand::Out { reg: _, late: _, place } => {
|
|
if let Some(place) = place {
|
|
outputs.push((asm_gen.stack_slots_output[i].unwrap(), *place));
|
|
}
|
|
}
|
|
CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => {
|
|
inputs.push((asm_gen.stack_slots_input[i].unwrap(), *in_value));
|
|
if let Some(out_place) = out_place {
|
|
outputs.push((asm_gen.stack_slots_output[i].unwrap(), *out_place));
|
|
}
|
|
}
|
|
CInlineAsmOperand::Const { value: _ } | CInlineAsmOperand::Symbol { symbol: _ } => {}
|
|
}
|
|
}
|
|
|
|
call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
|
|
}
|
|
|
|
pub(crate) fn codegen_naked_asm<'tcx>(
|
|
tcx: TyCtxt<'tcx>,
|
|
cx: &mut crate::CodegenCx,
|
|
module: &mut dyn Module,
|
|
instance: Instance<'tcx>,
|
|
span: Span,
|
|
symbol_name: &str,
|
|
template: &[InlineAsmTemplatePiece],
|
|
operands: &[InlineAsmOperand<'tcx>],
|
|
options: InlineAsmOptions,
|
|
) {
|
|
// FIXME add .eh_frame unwind info directives
|
|
|
|
let operands = operands
|
|
.iter()
|
|
.map(|operand| match *operand {
|
|
InlineAsmOperand::In { .. }
|
|
| InlineAsmOperand::Out { .. }
|
|
| InlineAsmOperand::InOut { .. } => {
|
|
span_bug!(span, "invalid operand type for naked asm")
|
|
}
|
|
InlineAsmOperand::Const { ref value } => {
|
|
let cv = instance.instantiate_mir_and_normalize_erasing_regions(
|
|
tcx,
|
|
ty::TypingEnv::fully_monomorphized(),
|
|
ty::EarlyBinder::bind(value.const_),
|
|
);
|
|
let const_value = cv
|
|
.eval(tcx, ty::TypingEnv::fully_monomorphized(), value.span)
|
|
.expect("erroneous constant missed by mono item collection");
|
|
|
|
let value = rustc_codegen_ssa::common::asm_const_to_str(
|
|
tcx,
|
|
span,
|
|
const_value,
|
|
RevealAllLayoutCx(tcx).layout_of(cv.ty()),
|
|
);
|
|
CInlineAsmOperand::Const { value }
|
|
}
|
|
InlineAsmOperand::SymFn { ref value } => {
|
|
if cfg!(not(feature = "inline_asm_sym")) {
|
|
tcx.dcx()
|
|
.span_err(span, "asm! and global_asm! sym operands are not yet supported");
|
|
}
|
|
|
|
let const_ = instance.instantiate_mir_and_normalize_erasing_regions(
|
|
tcx,
|
|
ty::TypingEnv::fully_monomorphized(),
|
|
ty::EarlyBinder::bind(value.const_),
|
|
);
|
|
if let ty::FnDef(def_id, args) = *const_.ty().kind() {
|
|
let instance = ty::Instance::resolve_for_fn_ptr(
|
|
tcx,
|
|
ty::TypingEnv::fully_monomorphized(),
|
|
def_id,
|
|
args,
|
|
)
|
|
.unwrap();
|
|
let symbol = tcx.symbol_name(instance);
|
|
|
|
// Pass a wrapper rather than the function itself as the function itself may not
|
|
// be exported from the main codegen unit and may thus be unreachable from the
|
|
// object file created by an external assembler.
|
|
let inline_asm_index = cx.inline_asm_index.get();
|
|
cx.inline_asm_index.set(inline_asm_index + 1);
|
|
let wrapper_name = format!(
|
|
"__inline_asm_{}_wrapper_n{}",
|
|
cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
|
|
inline_asm_index
|
|
);
|
|
let sig =
|
|
get_function_sig(tcx, module.target_config().default_call_conv, instance);
|
|
create_wrapper_function(module, sig, &wrapper_name, symbol.name);
|
|
|
|
CInlineAsmOperand::Symbol { symbol: wrapper_name }
|
|
} else {
|
|
span_bug!(span, "invalid type for asm sym (fn)");
|
|
}
|
|
}
|
|
InlineAsmOperand::SymStatic { def_id } => {
|
|
assert!(tcx.is_static(def_id));
|
|
let instance = Instance::mono(tcx, def_id);
|
|
CInlineAsmOperand::Symbol { symbol: tcx.symbol_name(instance).name.to_owned() }
|
|
}
|
|
InlineAsmOperand::Label { .. } => {
|
|
span_bug!(span, "asm! label operands are not yet supported");
|
|
}
|
|
})
|
|
.collect::<Vec<_>>();
|
|
|
|
let asm_gen = InlineAssemblyGenerator {
|
|
tcx,
|
|
arch: tcx.sess.asm_arch.unwrap(),
|
|
enclosing_def_id: instance.def_id(),
|
|
template,
|
|
operands: &operands,
|
|
options,
|
|
registers: Vec::new(),
|
|
stack_slots_clobber: Vec::new(),
|
|
stack_slots_input: Vec::new(),
|
|
stack_slots_output: Vec::new(),
|
|
stack_slot_size: Size::from_bytes(0),
|
|
is_naked: true,
|
|
};
|
|
|
|
let generated_asm = asm_gen.generate_asm_wrapper(symbol_name);
|
|
cx.global_asm.push_str(&generated_asm);
|
|
}
|
|
|
|
struct InlineAssemblyGenerator<'a, 'tcx> {
|
|
tcx: TyCtxt<'tcx>,
|
|
arch: InlineAsmArch,
|
|
enclosing_def_id: DefId,
|
|
template: &'a [InlineAsmTemplatePiece],
|
|
operands: &'a [CInlineAsmOperand<'tcx>],
|
|
options: InlineAsmOptions,
|
|
registers: Vec<Option<InlineAsmReg>>,
|
|
stack_slots_clobber: Vec<Option<Size>>,
|
|
stack_slots_input: Vec<Option<Size>>,
|
|
stack_slots_output: Vec<Option<Size>>,
|
|
stack_slot_size: Size,
|
|
is_naked: bool,
|
|
}
|
|
|
|
impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
|
|
fn allocate_registers(&mut self) {
|
|
assert!(!self.is_naked);
|
|
|
|
let sess = self.tcx.sess;
|
|
let map = allocatable_registers(
|
|
self.arch,
|
|
sess.relocation_model(),
|
|
self.tcx.asm_target_features(self.enclosing_def_id),
|
|
&sess.target,
|
|
);
|
|
let mut allocated = FxHashMap::<_, (bool, bool)>::default();
|
|
let mut regs = vec![None; self.operands.len()];
|
|
|
|
// Add explicit registers to the allocated set.
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
|
|
regs[i] = Some(reg);
|
|
allocated.entry(reg).or_default().0 = true;
|
|
}
|
|
CInlineAsmOperand::Out {
|
|
reg: InlineAsmRegOrRegClass::Reg(reg),
|
|
late: true,
|
|
..
|
|
} => {
|
|
regs[i] = Some(reg);
|
|
allocated.entry(reg).or_default().1 = true;
|
|
}
|
|
CInlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. }
|
|
| CInlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
|
|
regs[i] = Some(reg);
|
|
allocated.insert(reg, (true, true));
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
// Allocate out/inout/inlateout registers first because they are more constrained.
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::Out {
|
|
reg: InlineAsmRegOrRegClass::RegClass(class),
|
|
late: false,
|
|
..
|
|
}
|
|
| CInlineAsmOperand::InOut {
|
|
reg: InlineAsmRegOrRegClass::RegClass(class), ..
|
|
} => {
|
|
let mut alloc_reg = None;
|
|
for ® in &map[&class] {
|
|
let mut used = false;
|
|
reg.overlapping_regs(|r| {
|
|
if allocated.contains_key(&r) {
|
|
used = true;
|
|
}
|
|
});
|
|
|
|
if !used {
|
|
alloc_reg = Some(reg);
|
|
break;
|
|
}
|
|
}
|
|
|
|
let reg = alloc_reg.expect("cannot allocate registers");
|
|
regs[i] = Some(reg);
|
|
allocated.insert(reg, (true, true));
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
// Allocate in/lateout.
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => {
|
|
let mut alloc_reg = None;
|
|
for ® in &map[&class] {
|
|
let mut used = false;
|
|
reg.overlapping_regs(|r| {
|
|
if allocated.get(&r).copied().unwrap_or_default().0 {
|
|
used = true;
|
|
}
|
|
});
|
|
|
|
if !used {
|
|
alloc_reg = Some(reg);
|
|
break;
|
|
}
|
|
}
|
|
|
|
let reg = alloc_reg.expect("cannot allocate registers");
|
|
regs[i] = Some(reg);
|
|
allocated.entry(reg).or_default().0 = true;
|
|
}
|
|
CInlineAsmOperand::Out {
|
|
reg: InlineAsmRegOrRegClass::RegClass(class),
|
|
late: true,
|
|
..
|
|
} => {
|
|
let mut alloc_reg = None;
|
|
for ® in &map[&class] {
|
|
let mut used = false;
|
|
reg.overlapping_regs(|r| {
|
|
if allocated.get(&r).copied().unwrap_or_default().1 {
|
|
used = true;
|
|
}
|
|
});
|
|
|
|
if !used {
|
|
alloc_reg = Some(reg);
|
|
break;
|
|
}
|
|
}
|
|
|
|
let reg = alloc_reg.expect("cannot allocate registers");
|
|
regs[i] = Some(reg);
|
|
allocated.entry(reg).or_default().1 = true;
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
self.registers = regs;
|
|
}
|
|
|
|
fn allocate_stack_slots(&mut self) {
|
|
assert!(!self.is_naked);
|
|
|
|
let mut slot_size = Size::from_bytes(0);
|
|
let mut slots_clobber = vec![None; self.operands.len()];
|
|
let mut slots_input = vec![None; self.operands.len()];
|
|
let mut slots_output = vec![None; self.operands.len()];
|
|
|
|
let new_slot_fn = |slot_size: &mut Size, reg_class: InlineAsmRegClass| {
|
|
let reg_size =
|
|
reg_class.supported_types(self.arch).iter().map(|(ty, _)| ty.size()).max().unwrap();
|
|
let align = rustc_abi::Align::from_bytes(reg_size.bytes()).unwrap();
|
|
let offset = slot_size.align_to(align);
|
|
*slot_size = offset + reg_size;
|
|
offset
|
|
};
|
|
let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
|
|
|
|
// Allocate stack slots for saving clobbered registers
|
|
let abi_clobber = InlineAsmClobberAbi::parse(self.arch, &self.tcx.sess.target, sym::C)
|
|
.unwrap()
|
|
.clobbered_regs();
|
|
for (i, reg) in self.registers.iter().enumerate().filter_map(|(i, r)| r.map(|r| (i, r))) {
|
|
let mut need_save = true;
|
|
// If the register overlaps with a register clobbered by function call, then
|
|
// we don't need to save it.
|
|
for r in abi_clobber {
|
|
r.overlapping_regs(|r| {
|
|
if r == reg {
|
|
need_save = false;
|
|
}
|
|
});
|
|
|
|
if !need_save {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if need_save {
|
|
slots_clobber[i] = Some(new_slot(reg.reg_class()));
|
|
}
|
|
}
|
|
|
|
// Allocate stack slots for inout
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } => {
|
|
let slot = new_slot(reg.reg_class());
|
|
slots_input[i] = Some(slot);
|
|
slots_output[i] = Some(slot);
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
let slot_size_before_input = slot_size;
|
|
let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
|
|
|
|
// Allocate stack slots for input
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::In { reg, .. }
|
|
| CInlineAsmOperand::InOut { reg, out_place: None, .. } => {
|
|
slots_input[i] = Some(new_slot(reg.reg_class()));
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
// Reset slot size to before input so that input and output operands can overlap
|
|
// and save some memory.
|
|
let slot_size_after_input = slot_size;
|
|
slot_size = slot_size_before_input;
|
|
let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
|
|
|
|
// Allocate stack slots for output
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::Out { reg, place: Some(_), .. } => {
|
|
slots_output[i] = Some(new_slot(reg.reg_class()));
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
slot_size = slot_size.max(slot_size_after_input);
|
|
|
|
self.stack_slots_clobber = slots_clobber;
|
|
self.stack_slots_input = slots_input;
|
|
self.stack_slots_output = slots_output;
|
|
self.stack_slot_size = slot_size;
|
|
}
|
|
|
|
fn generate_asm_wrapper(&self, asm_name: &str) -> String {
|
|
let binary_format = crate::target_triple(self.tcx.sess).binary_format;
|
|
|
|
let mut generated_asm = String::new();
|
|
match binary_format {
|
|
BinaryFormat::Elf => {
|
|
writeln!(generated_asm, ".globl {}", asm_name).unwrap();
|
|
writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
|
|
writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
|
|
writeln!(generated_asm, "{}:", asm_name).unwrap();
|
|
}
|
|
BinaryFormat::Macho => {
|
|
writeln!(generated_asm, ".globl _{}", asm_name).unwrap();
|
|
writeln!(generated_asm, "_{}:", asm_name).unwrap();
|
|
}
|
|
BinaryFormat::Coff => {
|
|
writeln!(generated_asm, ".globl {}", asm_name).unwrap();
|
|
writeln!(generated_asm, "{}:", asm_name).unwrap();
|
|
}
|
|
_ => self
|
|
.tcx
|
|
.dcx()
|
|
.fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")),
|
|
}
|
|
|
|
let is_x86 = matches!(self.arch, InlineAsmArch::X86 | InlineAsmArch::X86_64);
|
|
|
|
if is_x86 {
|
|
generated_asm.push_str(".intel_syntax noprefix\n");
|
|
}
|
|
if !self.is_naked {
|
|
Self::prologue(&mut generated_asm, self.arch);
|
|
|
|
// Save clobbered registers
|
|
if !self.options.contains(InlineAsmOptions::NORETURN) {
|
|
for (reg, slot) in self
|
|
.registers
|
|
.iter()
|
|
.zip(self.stack_slots_clobber.iter().copied())
|
|
.filter_map(|(r, s)| r.zip(s))
|
|
{
|
|
Self::save_register(&mut generated_asm, self.arch, reg, slot);
|
|
}
|
|
}
|
|
|
|
// Write input registers
|
|
for (reg, slot) in self
|
|
.registers
|
|
.iter()
|
|
.zip(self.stack_slots_input.iter().copied())
|
|
.filter_map(|(r, s)| r.zip(s))
|
|
{
|
|
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
|
|
}
|
|
}
|
|
|
|
if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
|
|
generated_asm.push_str(".att_syntax\n");
|
|
}
|
|
|
|
// The actual inline asm
|
|
for piece in self.template {
|
|
match piece {
|
|
InlineAsmTemplatePiece::String(s) => {
|
|
generated_asm.push_str(s);
|
|
}
|
|
InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
|
|
match self.operands[*operand_idx] {
|
|
CInlineAsmOperand::In { .. }
|
|
| CInlineAsmOperand::Out { .. }
|
|
| CInlineAsmOperand::InOut { .. } => {
|
|
if self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
|
|
generated_asm.push('%');
|
|
}
|
|
|
|
let reg = self.registers[*operand_idx].unwrap();
|
|
match self.arch {
|
|
InlineAsmArch::X86_64 => match reg {
|
|
InlineAsmReg::X86(reg)
|
|
if reg as u32 >= X86InlineAsmReg::xmm0 as u32
|
|
&& reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
|
|
{
|
|
// rustc emits x0 rather than xmm0
|
|
let class = match *modifier {
|
|
None | Some('x') => "xmm",
|
|
Some('y') => "ymm",
|
|
Some('z') => "zmm",
|
|
_ => unreachable!(),
|
|
};
|
|
write!(
|
|
generated_asm,
|
|
"{class}{}",
|
|
reg as u32 - X86InlineAsmReg::xmm0 as u32
|
|
)
|
|
.unwrap();
|
|
}
|
|
_ => reg
|
|
.emit(&mut generated_asm, InlineAsmArch::X86_64, *modifier)
|
|
.unwrap(),
|
|
},
|
|
_ => reg.emit(&mut generated_asm, self.arch, *modifier).unwrap(),
|
|
}
|
|
}
|
|
CInlineAsmOperand::Const { ref value } => {
|
|
generated_asm.push_str(value);
|
|
}
|
|
CInlineAsmOperand::Symbol { ref symbol } => generated_asm.push_str(symbol),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
generated_asm.push('\n');
|
|
|
|
if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
|
|
generated_asm.push_str(".intel_syntax noprefix\n");
|
|
}
|
|
|
|
if !self.is_naked {
|
|
if !self.options.contains(InlineAsmOptions::NORETURN) {
|
|
// Read output registers
|
|
for (reg, slot) in self
|
|
.registers
|
|
.iter()
|
|
.zip(self.stack_slots_output.iter().copied())
|
|
.filter_map(|(r, s)| r.zip(s))
|
|
{
|
|
Self::save_register(&mut generated_asm, self.arch, reg, slot);
|
|
}
|
|
|
|
// Restore clobbered registers
|
|
for (reg, slot) in self
|
|
.registers
|
|
.iter()
|
|
.zip(self.stack_slots_clobber.iter().copied())
|
|
.filter_map(|(r, s)| r.zip(s))
|
|
{
|
|
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
|
|
}
|
|
|
|
Self::epilogue(&mut generated_asm, self.arch);
|
|
} else {
|
|
Self::epilogue_noreturn(&mut generated_asm, self.arch);
|
|
}
|
|
}
|
|
|
|
if is_x86 {
|
|
generated_asm.push_str(".att_syntax\n");
|
|
}
|
|
|
|
match binary_format {
|
|
BinaryFormat::Elf => {
|
|
writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap();
|
|
generated_asm.push_str(".text\n");
|
|
}
|
|
BinaryFormat::Macho | BinaryFormat::Coff => {}
|
|
_ => self
|
|
.tcx
|
|
.dcx()
|
|
.fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")),
|
|
}
|
|
|
|
generated_asm.push_str("\n\n");
|
|
|
|
generated_asm
|
|
}
|
|
|
|
fn prologue(generated_asm: &mut String, arch: InlineAsmArch) {
|
|
match arch {
|
|
InlineAsmArch::X86_64 => {
|
|
generated_asm.push_str(" push rbp\n");
|
|
generated_asm.push_str(" mov rbp,rsp\n");
|
|
generated_asm.push_str(" push rbx\n"); // rbx is callee saved
|
|
// rbx is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
|
|
generated_asm.push_str(" mov rbx,rdi\n");
|
|
}
|
|
InlineAsmArch::AArch64 => {
|
|
generated_asm.push_str(" stp fp, lr, [sp, #-32]!\n");
|
|
generated_asm.push_str(" mov fp, sp\n");
|
|
generated_asm.push_str(" str x19, [sp, #24]\n"); // x19 is callee saved
|
|
// x19 is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
|
|
generated_asm.push_str(" mov x19, x0\n");
|
|
}
|
|
InlineAsmArch::RiscV64 => {
|
|
generated_asm.push_str(" addi sp, sp, -16\n");
|
|
generated_asm.push_str(" sd ra, 8(sp)\n");
|
|
generated_asm.push_str(" sd s1, 0(sp)\n"); // s1 is callee saved
|
|
// s1/x9 is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
|
|
generated_asm.push_str(" mv s1, a0\n");
|
|
}
|
|
_ => unimplemented!("prologue for {:?}", arch),
|
|
}
|
|
}
|
|
|
|
fn epilogue(generated_asm: &mut String, arch: InlineAsmArch) {
|
|
match arch {
|
|
InlineAsmArch::X86_64 => {
|
|
generated_asm.push_str(" pop rbx\n");
|
|
generated_asm.push_str(" pop rbp\n");
|
|
generated_asm.push_str(" ret\n");
|
|
}
|
|
InlineAsmArch::AArch64 => {
|
|
generated_asm.push_str(" ldr x19, [sp, #24]\n");
|
|
generated_asm.push_str(" ldp fp, lr, [sp], #32\n");
|
|
generated_asm.push_str(" ret\n");
|
|
}
|
|
InlineAsmArch::RiscV64 => {
|
|
generated_asm.push_str(" ld s1, 0(sp)\n");
|
|
generated_asm.push_str(" ld ra, 8(sp)\n");
|
|
generated_asm.push_str(" addi sp, sp, 16\n");
|
|
generated_asm.push_str(" ret\n");
|
|
}
|
|
_ => unimplemented!("epilogue for {:?}", arch),
|
|
}
|
|
}
|
|
|
|
fn epilogue_noreturn(generated_asm: &mut String, arch: InlineAsmArch) {
|
|
match arch {
|
|
InlineAsmArch::X86_64 => {
|
|
generated_asm.push_str(" ud2\n");
|
|
}
|
|
InlineAsmArch::AArch64 => {
|
|
generated_asm.push_str(" brk #0x1\n");
|
|
}
|
|
InlineAsmArch::RiscV64 => {
|
|
generated_asm.push_str(" ebreak\n");
|
|
}
|
|
_ => unimplemented!("epilogue_noreturn for {:?}", arch),
|
|
}
|
|
}
|
|
|
|
fn save_register(
|
|
generated_asm: &mut String,
|
|
arch: InlineAsmArch,
|
|
reg: InlineAsmReg,
|
|
offset: Size,
|
|
) {
|
|
match arch {
|
|
InlineAsmArch::X86_64 => {
|
|
match reg {
|
|
InlineAsmReg::X86(reg)
|
|
if reg as u32 >= X86InlineAsmReg::xmm0 as u32
|
|
&& reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
|
|
{
|
|
// rustc emits x0 rather than xmm0
|
|
write!(generated_asm, " movups [rbx+0x{:x}], ", offset.bytes()).unwrap();
|
|
write!(generated_asm, "xmm{}", reg as u32 - X86InlineAsmReg::xmm0 as u32)
|
|
.unwrap();
|
|
}
|
|
_ => {
|
|
write!(generated_asm, " mov [rbx+0x{:x}], ", offset.bytes()).unwrap();
|
|
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
|
|
}
|
|
}
|
|
generated_asm.push('\n');
|
|
}
|
|
InlineAsmArch::AArch64 => {
|
|
generated_asm.push_str(" str ");
|
|
reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap();
|
|
writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap();
|
|
}
|
|
InlineAsmArch::RiscV64 => {
|
|
generated_asm.push_str(" sd ");
|
|
reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
|
|
writeln!(generated_asm, ", 0x{:x}(s1)", offset.bytes()).unwrap();
|
|
}
|
|
_ => unimplemented!("save_register for {:?}", arch),
|
|
}
|
|
}
|
|
|
|
fn restore_register(
|
|
generated_asm: &mut String,
|
|
arch: InlineAsmArch,
|
|
reg: InlineAsmReg,
|
|
offset: Size,
|
|
) {
|
|
match arch {
|
|
InlineAsmArch::X86_64 => {
|
|
match reg {
|
|
InlineAsmReg::X86(reg)
|
|
if reg as u32 >= X86InlineAsmReg::xmm0 as u32
|
|
&& reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
|
|
{
|
|
// rustc emits x0 rather than xmm0
|
|
write!(
|
|
generated_asm,
|
|
" movups xmm{}",
|
|
reg as u32 - X86InlineAsmReg::xmm0 as u32
|
|
)
|
|
.unwrap();
|
|
}
|
|
_ => {
|
|
generated_asm.push_str(" mov ");
|
|
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap()
|
|
}
|
|
}
|
|
writeln!(generated_asm, ", [rbx+0x{:x}]", offset.bytes()).unwrap();
|
|
}
|
|
InlineAsmArch::AArch64 => {
|
|
generated_asm.push_str(" ldr ");
|
|
reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap();
|
|
writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap();
|
|
}
|
|
InlineAsmArch::RiscV64 => {
|
|
generated_asm.push_str(" ld ");
|
|
reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
|
|
writeln!(generated_asm, ", 0x{:x}(s1)", offset.bytes()).unwrap();
|
|
}
|
|
_ => unimplemented!("restore_register for {:?}", arch),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn call_inline_asm<'tcx>(
|
|
fx: &mut FunctionCx<'_, '_, 'tcx>,
|
|
asm_name: &str,
|
|
slot_size: Size,
|
|
inputs: Vec<(Size, Value)>,
|
|
outputs: Vec<(Size, CPlace<'tcx>)>,
|
|
) {
|
|
let stack_slot = fx.create_stack_slot(u32::try_from(slot_size.bytes()).unwrap(), 16);
|
|
|
|
let inline_asm_func = fx
|
|
.module
|
|
.declare_function(asm_name, Linkage::Import, &Signature {
|
|
call_conv: CallConv::SystemV,
|
|
params: vec![AbiParam::new(fx.pointer_type)],
|
|
returns: vec![],
|
|
})
|
|
.unwrap();
|
|
let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, fx.bcx.func);
|
|
if fx.clif_comments.enabled() {
|
|
fx.add_comment(inline_asm_func, asm_name);
|
|
}
|
|
|
|
for (offset, value) in inputs {
|
|
stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).store(
|
|
fx,
|
|
value,
|
|
MemFlags::trusted(),
|
|
);
|
|
}
|
|
|
|
let stack_slot_addr = stack_slot.get_addr(fx);
|
|
fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);
|
|
|
|
for (offset, place) in outputs {
|
|
let ty = if place.layout().ty.is_simd() {
|
|
let (lane_count, lane_type) = place.layout().ty.simd_size_and_type(fx.tcx);
|
|
asm_clif_type(fx, lane_type).unwrap().by(lane_count.try_into().unwrap()).unwrap()
|
|
} else {
|
|
asm_clif_type(fx, place.layout().ty).unwrap()
|
|
};
|
|
let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load(
|
|
fx,
|
|
ty,
|
|
MemFlags::trusted(),
|
|
);
|
|
place.write_cvalue(fx, CValue::by_val(value, place.layout()));
|
|
}
|
|
}
|
|
|
|
fn asm_clif_type<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> Option<types::Type> {
|
|
match ty.kind() {
|
|
// Adapted from https://github.com/rust-lang/rust/blob/f3c66088610c1b80110297c2d9a8b5f9265b013f/compiler/rustc_hir_analysis/src/check/intrinsicck.rs#L136-L151
|
|
ty::Adt(adt, args) if fx.tcx.is_lang_item(adt.did(), LangItem::MaybeUninit) => {
|
|
let fields = &adt.non_enum_variant().fields;
|
|
let ty = fields[FieldIdx::from_u32(1)].ty(fx.tcx, args);
|
|
let ty::Adt(ty, args) = ty.kind() else {
|
|
unreachable!("expected first field of `MaybeUninit` to be an ADT")
|
|
};
|
|
assert!(
|
|
ty.is_manually_drop(),
|
|
"expected first field of `MaybeUninit` to be `ManuallyDrop`"
|
|
);
|
|
let fields = &ty.non_enum_variant().fields;
|
|
let ty = fields[FieldIdx::ZERO].ty(fx.tcx, args);
|
|
fx.clif_type(ty)
|
|
}
|
|
_ => fx.clif_type(ty),
|
|
}
|
|
}
|