diff --git a/compiler/rustc_codegen_gcc/src/type_of.rs b/compiler/rustc_codegen_gcc/src/type_of.rs index 5745acce6fee7..25b7a2e8c682b 100644 --- a/compiler/rustc_codegen_gcc/src/type_of.rs +++ b/compiler/rustc_codegen_gcc/src/type_of.rs @@ -1,6 +1,6 @@ use std::fmt::Write; -use gccjit::{Struct, Type}; +use gccjit::{RValue, Struct, Type}; use rustc_abi as abi; use rustc_abi::Primitive::*; use rustc_abi::{ @@ -373,7 +373,11 @@ impl<'gcc, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { unimplemented!(); } - fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> { + fn fn_decl_backend_type( + &self, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + _fn_ptr: RValue<'gcc>, + ) -> Type<'gcc> { // FIXME(antoyo): Should we do something with `FnAbiGcc::fn_attributes`? let FnAbiGcc { return_type, arguments_type, is_c_variadic, .. } = fn_abi.gcc_type(self); self.context.new_function_pointer_type(None, return_type, &arguments_type, is_c_variadic) diff --git a/compiler/rustc_codegen_llvm/messages.ftl b/compiler/rustc_codegen_llvm/messages.ftl index bda121c67fb62..2819b3bb6a244 100644 --- a/compiler/rustc_codegen_llvm/messages.ftl +++ b/compiler/rustc_codegen_llvm/messages.ftl @@ -3,6 +3,12 @@ codegen_llvm_autodiff_without_lto = using the autodiff feature requires using fa codegen_llvm_copy_bitcode = failed to copy bitcode to object file: {$err} +codegen_llvm_deprecated_intrinsic = + Using deprecated intrinsic `{$name}`, consider using other intrinsics/instructions + +codegen_llvm_deprecated_intrinsic_with_replacement = + Using deprecated intrinsic `{$name}`, `{$replacement}` can be used instead + codegen_llvm_dynamic_linking_with_lto = cannot prefer dynamic linking when performing LTO .note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO @@ -19,6 +25,15 @@ codegen_llvm_from_llvm_diag = {$message} codegen_llvm_from_llvm_optimization_diag = {$filename}:{$line}:{$column} {$pass_name} ({$kind}): {$message} +codegen_llvm_intrinsic_signature_mismatch = + Intrinsic signature mismatch for `{$name}`: expected signature `{$llvm_fn_ty}`, found `{$rust_fn_ty}` + +codegen_llvm_intrinsic_wrong_arch = + Intrinsic `{$name}` cannot be used with target arch `{$target_arch}` + +codegen_llvm_invalid_intrinsic = + Invalid LLVM Intrinsic `{$name}` + codegen_llvm_load_bitcode = failed to load bitcode of module "{$name}" codegen_llvm_load_bitcode_with_llvm_err = failed to load bitcode of module "{$name}": {$llvm_err} diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 119cd634f9827..d8e867ea29fa0 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -1,5 +1,5 @@ use std::borrow::Borrow; -use std::cmp; +use std::{cmp, iter}; use libc::c_uint; use rustc_abi::{ @@ -7,6 +7,7 @@ use rustc_abi::{ X86Call, }; use rustc_codegen_ssa::MemFlags; +use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; @@ -22,7 +23,7 @@ use smallvec::SmallVec; use crate::attributes::{self, llfn_attrs_from_instance}; use crate::builder::Builder; -use crate::context::CodegenCx; +use crate::context::{CodegenCx, GenericCx, SCx}; use crate::llvm::{self, Attribute, AttributePlace}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; @@ -300,8 +301,48 @@ impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } +pub(crate) enum FunctionSignature<'ll> { + /// This is an LLVM intrinsic, the signature is obtained directly from LLVM, and **may not match the Rust signature** + LLVMSignature(llvm::Intrinsic, &'ll Type), + /// This is an LLVM intrinsic, but the signature is just the Rust signature. + /// FIXME: this should ideally not exist, we should be using the LLVM signature for all LLVM intrinsics + RustSignature(llvm::Intrinsic, &'ll Type), + /// The name starts with `llvm.`, but can't obtain the intrinsic ID. May be invalid or upgradable + MaybeInvalid(&'ll Type), + /// Just the Rust signature + NotIntrinsic(&'ll Type), +} + +impl<'ll> FunctionSignature<'ll> { + pub(crate) fn fn_ty(&self) -> &'ll Type { + match self { + FunctionSignature::LLVMSignature(_, fn_ty) + | FunctionSignature::RustSignature(_, fn_ty) + | FunctionSignature::MaybeInvalid(fn_ty) + | FunctionSignature::NotIntrinsic(fn_ty) => fn_ty, + } + } + + pub(crate) fn intrinsic(&self) -> Option { + match self { + FunctionSignature::RustSignature(intrinsic, _) + | FunctionSignature::LLVMSignature(intrinsic, _) => Some(*intrinsic), + _ => None, + } + } +} + pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> { - fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type>; + /// When `do_verify` is set, this function performs checks for the signature of LLVM intrinsics + /// and emits a fatal error if it doesn't match. These checks are important,but somewhat expensive + /// So they are only used at function definitions, not at callsites + fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>, name: &[u8]) -> FunctionSignature<'ll>; + /// The normal Rust signature for this + fn rust_signature(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + /// **If this function is an LLVM intrinsic** checks if the LLVM signature provided matches with this + fn verify_intrinsic_signature(&self, cx: &CodegenCx<'ll, 'tcx>, llvm_ty: &'ll Type) -> bool; fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv; @@ -314,30 +355,97 @@ pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> { ); /// Apply attributes to a function call. - fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value); + fn apply_attrs_callsite( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + callsite: &'ll Value, + llfn: &'ll Value, + ); +} + +impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { + pub(crate) fn equate_ty(&self, rust_ty: &'ll Type, llvm_ty: &'ll Type) -> bool { + if rust_ty == llvm_ty { + return true; + } + + match self.type_kind(llvm_ty) { + TypeKind::X86_AMX if self.type_kind(rust_ty) == TypeKind::Vector => { + let element_count = self.vector_length(rust_ty); + let element_ty = self.element_type(rust_ty); + + let element_size_bits = match self.type_kind(element_ty) { + TypeKind::Half => 16, + TypeKind::Float => 32, + TypeKind::Double => 64, + TypeKind::FP128 => 128, + TypeKind::Integer => self.int_width(element_ty), + TypeKind::Pointer => self.int_width(self.isize_ty()), + _ => bug!( + "Vector element type `{element_ty:?}` not one of integer, float or pointer" + ), + }; + let vector_size_bits = element_size_bits * element_count as u64; + + vector_size_bits == 8192 + } + TypeKind::BFloat => rust_ty == self.type_i16(), + TypeKind::Vector => { + let llvm_element_count = self.vector_length(llvm_ty) as u64; + let llvm_element_ty = self.element_type(llvm_ty); + + if llvm_element_ty == self.type_bf16() { + rust_ty == self.type_vector(self.type_i16(), llvm_element_count) + } else if llvm_element_ty == self.type_i1() { + let int_width = cmp::max(llvm_element_count.next_power_of_two(), 8); + rust_ty == self.type_ix(int_width) + } else { + false + } + } + TypeKind::Struct if self.type_kind(rust_ty) == TypeKind::Struct => { + let rust_element_tys = self.struct_element_types(rust_ty); + let llvm_element_tys = self.struct_element_types(llvm_ty); + + if rust_element_tys.len() != llvm_element_tys.len() { + return false; + } + + iter::zip(rust_element_tys, llvm_element_tys).all( + |(rust_element_ty, llvm_element_ty)| { + self.equate_ty(rust_element_ty, llvm_element_ty) + }, + ) + } + _ => false, + } + } } impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { - fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + match &self.ret.mode { + PassMode::Ignore => cx.type_void(), + PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), + PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx), + PassMode::Indirect { .. } => cx.type_void(), + } + } + + fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type> { + let indirect_return = matches!(self.ret.mode, PassMode::Indirect { .. }); + // Ignore "extra" args from the call site for C variadic functions. // Only the "fixed" args are part of the LLVM function signature. let args = if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args }; - // This capacity calculation is approximate. - let mut llargument_tys = Vec::with_capacity( - self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 }, - ); + let mut llargument_tys = + Vec::with_capacity(args.len() + if indirect_return { 1 } else { 0 }); - let llreturn_ty = match &self.ret.mode { - PassMode::Ignore => cx.type_void(), - PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), - PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx), - PassMode::Indirect { .. } => { - llargument_tys.push(cx.type_ptr()); - cx.type_void() - } - }; + if indirect_return { + llargument_tys.push(cx.type_ptr()); + } for arg in args { // Note that the exact number of arguments pushed here is carefully synchronized with @@ -384,10 +492,57 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { llargument_tys.push(llarg_ty); } + llargument_tys + } + + fn verify_intrinsic_signature(&self, cx: &CodegenCx<'ll, 'tcx>, llvm_fn_ty: &'ll Type) -> bool { + let rust_return_ty = self.llvm_return_type(cx); + let rust_argument_tys = self.llvm_argument_types(cx); + + let llvm_return_ty = cx.get_return_type(llvm_fn_ty); + let llvm_argument_tys = cx.func_params_types(llvm_fn_ty); + let llvm_is_variadic = cx.func_is_variadic(llvm_fn_ty); + + if self.c_variadic != llvm_is_variadic || rust_argument_tys.len() != llvm_argument_tys.len() + { + return false; + } + + iter::once((rust_return_ty, llvm_return_ty)) + .chain(iter::zip(rust_argument_tys, llvm_argument_tys)) + .all(|(rust_ty, llvm_ty)| cx.equate_ty(rust_ty, llvm_ty)) + } + + fn rust_signature(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + let return_ty = self.llvm_return_type(cx); + let argument_tys = self.llvm_argument_types(cx); + if self.c_variadic { - cx.type_variadic_func(&llargument_tys, llreturn_ty) + cx.type_variadic_func(&argument_tys, return_ty) + } else { + cx.type_func(&argument_tys, return_ty) + } + } + + fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>, name: &[u8]) -> FunctionSignature<'ll> { + if name.starts_with(b"llvm.") { + if let Some(intrinsic) = llvm::Intrinsic::lookup(name) { + if !intrinsic.is_overloaded() { + // FIXME: also do this for overloaded intrinsics + FunctionSignature::LLVMSignature(intrinsic, cx.intrinsic_type(intrinsic, &[])) + } else { + FunctionSignature::RustSignature(intrinsic, self.rust_signature(cx)) + } + } else { + // it's one of 2 cases, + // - either the base name is invalid + // - it has been superceded by something else, so the intrinsic was removed entirely + // to check for upgrades, we need the `llfn`, so we defer it for now + + FunctionSignature::MaybeInvalid(self.rust_signature(cx)) + } } else { - cx.type_func(&llargument_tys, llreturn_ty) + FunctionSignature::NotIntrinsic(self.rust_signature(cx)) } } @@ -530,7 +685,17 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } } - fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) { + fn apply_attrs_callsite( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + callsite: &'ll Value, + llfn: &'ll Value, + ) { + // Don't apply any attributes to LLVM intrinsics, they will be applied by AutoUpgrade + if llvm::get_value_name(llfn).starts_with(b"llvm.") { + return; + } + let mut func_attrs = SmallVec::<[_; 2]>::new(); if self.ret.layout.is_uninhabited() { func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx)); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index ec006b591929d..7ed585d4d8540 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1,6 +1,6 @@ use std::borrow::{Borrow, Cow}; use std::ops::Deref; -use std::{iter, ptr}; +use std::{cmp, iter, ptr}; pub(crate) mod autodiff; @@ -67,7 +67,6 @@ impl<'a, 'll> SBuilder<'a, 'll> { ) -> &'ll Value { debug!("call {:?} with args ({:?})", llfn, args); - let args = self.check_call("call", llty, llfn, args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { @@ -349,7 +348,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ) -> &'ll Value { debug!("invoke {:?} with args ({:?})", llfn, args); - let args = self.check_call("invoke", llty, llfn, args); + let args = self.cast_arguments("invoke", llty, llfn, args, fn_abi.is_some()); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { @@ -380,9 +379,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ) }; if let Some(fn_abi) = fn_abi { - fn_abi.apply_attrs_callsite(self, invoke); + fn_abi.apply_attrs_callsite(self, invoke, llfn); + self.cast_return(fn_abi, llfn, invoke) + } else { + invoke } - invoke } fn unreachable(&mut self) { @@ -1404,7 +1405,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ) -> &'ll Value { debug!("call {:?} with args ({:?})", llfn, args); - let args = self.check_call("call", llty, llfn, args); + let args = self.cast_arguments("call", llty, llfn, args, fn_abi.is_some()); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { @@ -1433,9 +1434,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ) }; if let Some(fn_abi) = fn_abi { - fn_abi.apply_attrs_callsite(self, call); + fn_abi.apply_attrs_callsite(self, call, llfn); + self.cast_return(fn_abi, llfn, call) + } else { + call } - call } fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { @@ -1602,47 +1605,6 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { ret.expect("LLVM does not have support for catchret") } - fn check_call<'b>( - &mut self, - typ: &str, - fn_ty: &'ll Type, - llfn: &'ll Value, - args: &'b [&'ll Value], - ) -> Cow<'b, [&'ll Value]> { - assert!( - self.cx.type_kind(fn_ty) == TypeKind::Function, - "builder::{typ} not passed a function, but {fn_ty:?}" - ); - - let param_tys = self.cx.func_params_types(fn_ty); - - let all_args_match = iter::zip(¶m_tys, args.iter().map(|&v| self.cx.val_ty(v))) - .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty); - - if all_args_match { - return Cow::Borrowed(args); - } - - let casted_args: Vec<_> = iter::zip(param_tys, args) - .enumerate() - .map(|(i, (expected_ty, &actual_val))| { - let actual_ty = self.cx.val_ty(actual_val); - if expected_ty != actual_ty { - debug!( - "type mismatch in function call of {:?}. \ - Expected {:?} for param {}, got {:?}; injecting bitcast", - llfn, expected_ty, i, actual_ty - ); - self.bitcast(actual_val, expected_ty) - } else { - actual_val - } - }) - .collect(); - - Cow::Owned(casted_args) - } - pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } } @@ -1714,6 +1676,165 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None, None) } + fn trunc_int_to_i1_vector(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + let vector_length = self.vector_length(dest_ty) as u64; + let int_width = cmp::max(vector_length.next_power_of_two(), 8); + + let bitcasted = self.bitcast(val, self.type_vector(self.type_i1(), int_width)); + if vector_length == int_width { + bitcasted + } else { + let shuffle_mask = + (0..vector_length).map(|i| self.const_i32(i as i32)).collect::>(); + self.shuffle_vector(bitcasted, bitcasted, self.const_vector(&shuffle_mask)) + } + } + + fn zext_i1_vector_to_int( + &mut self, + mut val: &'ll Value, + src_ty: &'ll Type, + dest_ty: &'ll Type, + ) -> &'ll Value { + let vector_length = self.vector_length(src_ty) as u64; + let int_width = cmp::max(vector_length.next_power_of_two(), 8); + + if vector_length != int_width { + let shuffle_indices = match vector_length { + 0 => unreachable!("zero length vectors are not allowed"), + 1 => vec![0, 1, 1, 1, 1, 1, 1, 1], + 2 => vec![0, 1, 2, 3, 2, 3, 2, 3], + 3 => vec![0, 1, 2, 3, 4, 5, 3, 4], + 4.. => (0..int_width as i32).collect(), + }; + let shuffle_mask = + shuffle_indices.into_iter().map(|i| self.const_i32(i)).collect::>(); + val = + self.shuffle_vector(val, self.const_null(src_ty), self.const_vector(&shuffle_mask)); + } + + self.bitcast(val, dest_ty) + } + + fn autocast( + &mut self, + llfn: &'ll Value, + val: &'ll Value, + src_ty: &'ll Type, + dest_ty: &'ll Type, + is_argument: bool, + ) -> &'ll Value { + let (rust_ty, llvm_ty) = if is_argument { (src_ty, dest_ty) } else { (dest_ty, src_ty) }; + + if rust_ty == llvm_ty { + return val; + } + + match self.type_kind(llvm_ty) { + TypeKind::X86_AMX => { + let vector_length = self.vector_length(rust_ty); + let element_ty = self.element_type(rust_ty); + let element_ty_str = match self.type_kind(element_ty) { + TypeKind::Half => "f16", + TypeKind::Float => "f32", + TypeKind::Double => "f64", + TypeKind::FP128 => "f128", + TypeKind::Integer => &format!("i{}", self.int_width(element_ty)), + TypeKind::Pointer => "p0", + _ => bug!( + "Vector element type `{element_ty:?}` not one of integer, float or pointer" + ), + }; + + let base_name = if is_argument { + "llvm.x86.cast.vector.to.tile" + } else { + "llvm.x86.cast.tile.to.vector" + }; + let llvm_intrinsic = format!("{base_name}.v{vector_length}{element_ty_str}"); + let fn_ty = self.type_func(&[src_ty], dest_ty); + let f = self.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + self.call(fn_ty, None, None, f, &[val], None, None) + } + TypeKind::Vector if self.element_type(llvm_ty) == self.type_i1() => { + if is_argument { + self.trunc_int_to_i1_vector(val, dest_ty) + } else { + self.zext_i1_vector_to_int(val, src_ty, dest_ty) + } + } + TypeKind::Struct => { + let mut ret = self.const_poison(dest_ty); + for (idx, (src_element_ty, dest_element_ty)) in + iter::zip(self.struct_element_types(src_ty), self.struct_element_types(dest_ty)) + .enumerate() + { + let elt = self.extract_value(val, idx as u64); + let casted_elt = + self.autocast(llfn, elt, src_element_ty, dest_element_ty, is_argument); + ret = self.insert_value(ret, casted_elt, idx as u64); + } + ret + } + _ => self.bitcast(val, dest_ty), // for `bf16(xN)` <-> `u16(xN)` + } + } + + fn cast_arguments<'b>( + &mut self, + typ: &str, + fn_ty: &'ll Type, + llfn: &'ll Value, + args: &'b [&'ll Value], + has_fnabi: bool, + ) -> Cow<'b, [&'ll Value]> { + assert_eq!( + self.type_kind(fn_ty), + TypeKind::Function, + "{typ} not passed a function, but {fn_ty:?}" + ); + + let param_tys = self.func_params_types(fn_ty); + + let mut casted_args = Cow::Borrowed(args); + + for (idx, (dest_ty, &arg)) in iter::zip(param_tys, args).enumerate() { + let src_ty = self.val_ty(arg); + assert!( + self.equate_ty(src_ty, dest_ty), + "Cannot match `{dest_ty:?}` (expected) with `{src_ty:?}` (found) in `{llfn:?}`" + ); + + let casted_arg = self.autocast(llfn, arg, src_ty, dest_ty, true); + if arg != casted_arg { + assert!( + has_fnabi, + "Should inject autocasts in function call of {llfn:?}, but not able to get Rust signature" + ); + + casted_args.to_mut()[idx] = casted_arg; + } + } + + casted_args + } + + fn cast_return( + &mut self, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + llfn: &'ll Value, + ret: &'ll Value, + ) -> &'ll Value { + let src_ty = self.val_ty(ret); + let dest_ty = fn_abi.llvm_return_type(self); + assert!( + self.equate_ty(dest_ty, src_ty), + "Cannot match `{src_ty:?}` (expected) with `{dest_ty:?}` (found) in `{llfn:?}`" + ); + + self.autocast(llfn, ret, src_ty, dest_ty, false) + } + pub(crate) fn landing_pad( &mut self, ty: &'ll Type, @@ -1743,7 +1864,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { ) -> &'ll Value { debug!("invoke {:?} with args ({:?})", llfn, args); - let args = self.check_call("callbr", llty, llfn, args); + let args = self.cast_arguments("callbr", llty, llfn, args, fn_abi.is_some()); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { @@ -1775,9 +1896,11 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { ) }; if let Some(fn_abi) = fn_abi { - fn_abi.apply_attrs_callsite(self, callbr); + fn_abi.apply_attrs_callsite(self, callbr, llfn); + self.cast_return(fn_abi, llfn, callbr) + } else { + callbr } - callbr } // Emits CFI pointer type membership tests. diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 8d6e1d8941b72..30c093657908e 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -1264,6 +1264,21 @@ impl<'ll> CodegenCx<'ll, '_> { self.eh_catch_typeinfo.set(Some(eh_catch_typeinfo)); eh_catch_typeinfo } + + pub(crate) fn intrinsic_type( + &self, + intrinsic: llvm::Intrinsic, + type_params: &[&'ll Type], + ) -> &'ll Type { + unsafe { + llvm::LLVMIntrinsicGetType( + self.llcx(), + intrinsic.id(), + type_params.as_ptr(), + type_params.len(), + ) + } + } } impl CodegenCx<'_, '_> { diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 2419ec1f88854..447b453ccaca9 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -22,14 +22,14 @@ use rustc_target::callconv::FnAbi; use smallvec::SmallVec; use tracing::debug; -use crate::abi::FnAbiLlvmExt; +use crate::abi::{FnAbiLlvmExt, FunctionSignature}; use crate::common::AsCCharPtr; use crate::context::{CodegenCx, GenericCx, SCx, SimpleCx}; use crate::llvm::AttributePlace::Function; use crate::llvm::Visibility; use crate::type_::Type; use crate::value::Value; -use crate::{attributes, llvm}; +use crate::{attributes, errors, llvm}; /// Declare a function with a SimpleCx. /// @@ -150,6 +150,22 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { ) -> &'ll Value { debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi); + let signature = fn_abi.llvm_type(self, name.as_bytes()); + + let span = || instance.map(|instance| self.tcx.def_span(instance.def_id())); + + if let FunctionSignature::LLVMSignature(_, llvm_fn_ty) = signature { + // check if the intrinsic signatures match + if !fn_abi.verify_intrinsic_signature(self, llvm_fn_ty) { + self.tcx.dcx().emit_fatal(errors::IntrinsicSignatureMismatch { + name, + llvm_fn_ty: &format!("{llvm_fn_ty:?}"), + rust_fn_ty: &format!("{:?}", fn_abi.rust_signature(self)), + span: span(), + }); + } + } + // Function addresses in Rust are never significant, allowing functions to // be merged. let llfn = declare_raw_fn( @@ -158,9 +174,66 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { fn_abi.llvm_cconv(self), llvm::UnnamedAddr::Global, llvm::Visibility::Default, - fn_abi.llvm_type(self), + signature.fn_ty(), ); - fn_abi.apply_attrs_llfn(self, llfn, instance); + + if let Some(intrinsic) = signature.intrinsic() { + if intrinsic.is_target_specific() { + let (llvm_arch, _) = name[5..].split_once('.').unwrap(); + let target_arch = self.tcx.sess.target.arch.as_ref(); + + let is_correct_arch = match llvm_arch { + "aarch64" => matches!(target_arch, "aarch64" | "arm64ec"), + "amdgcn" => target_arch == "amdgpu", + "arm" | "bpf" | "hexagon" => target_arch == llvm_arch, + "loongarch" => matches!(target_arch, "loongarch32" | "loongarch64"), + "mips" => target_arch.starts_with("mips"), + "nvvm" => target_arch == "nvptx64", + "ppc" => matches!(target_arch, "powerpc" | "powerpc64"), + "riscv" => matches!(target_arch, "riscv32" | "riscv64"), + "s390" => target_arch == "s390x", + "spv" => target_arch == "spirv", + "wasm" => matches!(target_arch, "wasm32" | "wasm64"), + "x86" => matches!(target_arch, "x86" | "x86_64"), + _ => true, // fallback for unknown archs + }; + + if !is_correct_arch { + self.tcx.dcx().emit_fatal(errors::IntrinsicWrongArch { + name, + target_arch, + span: span(), + }); + } + } + } else { + // Don't apply any attributes to intrinsics, they will be applied by AutoUpgrade + fn_abi.apply_attrs_llfn(self, llfn, instance); + } + + if let FunctionSignature::MaybeInvalid(..) = signature { + let mut new_llfn = None; + let can_upgrade = + unsafe { llvm::LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn, false) }; + + if can_upgrade { + // not all intrinsics are upgraded to some other intrinsics, most are upgraded to instruction sequences + if let Some(new_llfn) = new_llfn { + self.tcx.dcx().emit_note(errors::DeprecatedIntrinsicWithReplacement { + name, + replacement: str::from_utf8(llvm::get_value_name(new_llfn)).unwrap(), + span: span(), + }); + } else if self.tcx.sess.opts.verbose { + // At least for now, we are only emitting notes for deprecated intrinsics with no direct replacement + // because they are used quite a lot in stdarch. After the stdarch uses has been removed, we can make + // this always emit a note (or even an warning) + self.tcx.dcx().emit_note(errors::DeprecatedIntrinsic { name, span: span() }); + } + } else { + self.tcx.dcx().emit_fatal(errors::InvalidIntrinsic { name, span: span() }); + } + } if self.tcx.sess.is_sanitizer_cfi_enabled() { if let Some(instance) = instance { diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index eaafc68071291..31c79b00d2618 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -209,3 +209,47 @@ pub(crate) struct FixedX18InvalidArch<'a> { #[derive(Diagnostic)] #[diag(codegen_llvm_sanitizer_kcfi_arity_requires_llvm_21_0_0)] pub(crate) struct SanitizerKcfiArityRequiresLLVM2100; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_intrinsic_signature_mismatch)] +pub(crate) struct IntrinsicSignatureMismatch<'a> { + pub name: &'a str, + pub llvm_fn_ty: &'a str, + pub rust_fn_ty: &'a str, + #[primary_span] + pub span: Option, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_invalid_intrinsic)] +pub(crate) struct InvalidIntrinsic<'a> { + pub name: &'a str, + #[primary_span] + pub span: Option, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_deprecated_intrinsic)] +pub(crate) struct DeprecatedIntrinsic<'a> { + pub name: &'a str, + #[primary_span] + pub span: Option, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_deprecated_intrinsic_with_replacement)] +pub(crate) struct DeprecatedIntrinsicWithReplacement<'a> { + pub name: &'a str, + pub replacement: &'a str, + #[primary_span] + pub span: Option, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_intrinsic_wrong_arch)] +pub(crate) struct IntrinsicWrongArch<'a> { + pub name: &'a str, + pub target_arch: &'a str, + #[primary_span] + pub span: Option, +} diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 10697b9a71f9c..7b667bdd7dc57 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1067,7 +1067,7 @@ fn gen_fn<'a, 'll, 'tcx>( codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>), ) -> (&'ll Type, &'ll Value) { let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); - let llty = fn_abi.llvm_type(cx); + let llty = fn_abi.llvm_type(cx, name.as_bytes()).fn_ty(); let llfn = cx.declare_fn(name, fn_abi, None); cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index e27fbf94f341d..f1a1dff8b7618 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -15,6 +15,7 @@ use std::fmt::Debug; use std::marker::PhantomData; +use std::num::NonZero; use std::ptr; use bitflags::bitflags; @@ -1050,6 +1051,9 @@ unsafe extern "C" { pub(crate) fn LLVMDoubleTypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMFP128TypeInContext(C: &Context) -> &Type; + // Operations on non-IEEE real types + pub(crate) fn LLVMBFloatTypeInContext(C: &Context) -> &Type; + // Operations on function types pub(crate) fn LLVMFunctionType<'a>( ReturnType: &'a Type, @@ -1059,6 +1063,7 @@ unsafe extern "C" { ) -> &'a Type; pub(crate) fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint; pub(crate) fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type); + pub(crate) fn LLVMIsFunctionVarArg(FunctionTy: &Type) -> Bool; // Operations on struct types pub(crate) fn LLVMStructTypeInContext<'a>( @@ -1195,6 +1200,22 @@ unsafe extern "C" { // Operations on functions pub(crate) fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); + // Operations about llvm intrinsics + pub(crate) fn LLVMLookupIntrinsicID(Name: *const c_char, NameLen: size_t) -> c_uint; + pub(crate) fn LLVMIntrinsicGetType<'a>( + C: &'a Context, + ID: NonZero, + ParamTypes: *const &'a Type, + ParamCount: size_t, + ) -> &'a Type; + pub(crate) fn LLVMIntrinsicIsOverloaded(ID: NonZero) -> Bool; + pub(crate) fn LLVMRustUpgradeIntrinsicFunction<'a>( + Fn: &'a Value, + NewFn: &mut Option<&'a Value>, + CanUpgradeDebugIntrinsicsToRecords: bool, + ) -> bool; + pub(crate) fn LLVMRustIsTargetIntrinsic(ID: NonZero) -> bool; + // Operations on parameters pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; pub(crate) safe fn LLVMCountParams(Fn: &Value) -> c_uint; @@ -1714,6 +1735,9 @@ unsafe extern "C" { Packed: Bool, ); + pub(crate) fn LLVMCountStructElementTypes(StructTy: &Type) -> c_uint; + pub(crate) fn LLVMGetStructElementTypes<'a>(StructTy: &'a Type, Dest: *mut &'a Type); + pub(crate) safe fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; pub(crate) fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index ed23f91193013..e70351ff7fd31 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -1,6 +1,7 @@ #![allow(non_snake_case)] use std::ffi::{CStr, CString}; +use std::num::NonZero; use std::ptr; use std::str::FromStr; use std::string::FromUtf8Error; @@ -327,6 +328,30 @@ pub(crate) fn get_value_name(value: &Value) -> &[u8] { } } +#[derive(Debug, Copy, Clone)] +pub(crate) struct Intrinsic { + id: NonZero, +} + +impl Intrinsic { + pub(crate) fn lookup(name: &[u8]) -> Option { + let id = unsafe { LLVMLookupIntrinsicID(name.as_c_char_ptr(), name.len()) }; + NonZero::new(id).map(|id| Self { id }) + } + + pub(crate) fn id(self) -> NonZero { + self.id + } + + pub(crate) fn is_overloaded(self) -> bool { + unsafe { LLVMIntrinsicIsOverloaded(self.id) == True } + } + + pub(crate) fn is_target_specific(self) -> bool { + unsafe { LLVMRustIsTargetIntrinsic(self.id) } + } +} + /// Safe wrapper for `LLVMSetValueName2` from a byte slice pub(crate) fn set_value_name(value: &Value, name: &[u8]) { unsafe { diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 169036f515298..d258b3f149a28 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -75,6 +75,20 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { args } } + + pub(crate) fn func_is_variadic(&self, ty: &'ll Type) -> bool { + unsafe { llvm::LLVMIsFunctionVarArg(ty) == True } + } + + pub(crate) fn struct_element_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { + unsafe { + let n_args = llvm::LLVMCountStructElementTypes(ty) as usize; + let mut args = Vec::with_capacity(n_args); + llvm::LLVMGetStructElementTypes(ty, args.as_mut_ptr()); + args.set_len(n_args); + args + } + } } impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { pub(crate) fn type_bool(&self) -> &'ll Type { @@ -154,6 +168,10 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { ) } } + + pub(crate) fn type_bf16(&self) -> &'ll Type { + unsafe { llvm::LLVMBFloatTypeInContext(self.llcx()) } + } } impl<'ll, CX: Borrow>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { @@ -227,7 +245,7 @@ impl<'ll, CX: Borrow>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { fn float_width(&self, ty: &'ll Type) -> usize { match self.type_kind(ty) { - TypeKind::Half => 16, + TypeKind::Half | TypeKind::BFloat => 16, TypeKind::Float => 32, TypeKind::Double => 64, TypeKind::X86_FP80 => 80, @@ -284,8 +302,12 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn cast_backend_type(&self, ty: &CastTarget) -> &'ll Type { ty.llvm_type(self) } - fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { - fn_abi.llvm_type(self) + fn fn_decl_backend_type( + &self, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + fn_ptr: &'ll Value, + ) -> &'ll Type { + fn_abi.llvm_type(self, llvm::get_value_name(fn_ptr)).fn_ty() } fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { fn_abi.ptr_to_llvm_type(self) diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 43b87171d510d..993b918ab2e9b 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -187,7 +187,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { // If there is a cleanup block and the function we're calling can unwind, then // do an invoke, otherwise do a call. - let fn_ty = bx.fn_decl_backend_type(fn_abi); + let fn_ty = bx.fn_decl_backend_type(fn_abi, fn_ptr); let fn_attrs = if bx.tcx().def_kind(fx.instance.def_id()).has_codegen_attrs() { Some(bx.tcx().codegen_fn_attrs(fx.instance.def_id())) @@ -1835,7 +1835,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if is_call_from_compiler_builtins_to_upstream_monomorphization(bx.tcx(), instance) { bx.abort(); } else { - let fn_ty = bx.fn_decl_backend_type(fn_abi); + let fn_ty = bx.fn_decl_backend_type(fn_abi, fn_ptr); let llret = bx.call(fn_ty, None, Some(fn_abi), fn_ptr, &[], funclet.as_ref(), None); bx.apply_attrs_to_cleanup_callsite(llret); diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 5c14fe5cd10b7..48101cf6664a6 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -779,7 +779,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let fn_ptr = bx.get_fn_addr(instance); let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty()); - let fn_ty = bx.fn_decl_backend_type(fn_abi); + let fn_ty = bx.fn_decl_backend_type(fn_abi, fn_ptr); let fn_attrs = if bx.tcx().def_kind(instance.def_id()).has_codegen_attrs() { Some(bx.tcx().codegen_fn_attrs(instance.def_id())) } else { diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 577012151e49f..778440853987d 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -68,7 +68,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // Generate the call. Cannot use `do_call` since we don't have a MIR terminator so we // can't create a `TerminationCodegenHelper`. (But we are in good company, this code is // duplicated plenty of times.) - let fn_ty = bx.fn_decl_backend_type(fn_abi); + let fn_ty = bx.fn_decl_backend_type(fn_abi, llfn); bx.call( fn_ty, diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index c3fc21a92854a..4c3e057f6f90c 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -96,7 +96,11 @@ pub trait LayoutTypeCodegenMethods<'tcx>: BackendTypes { /// such as when it's stack-allocated or when it's being loaded or stored. fn backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type; fn cast_backend_type(&self, ty: &CastTarget) -> Self::Type; - fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; + fn fn_decl_backend_type( + &self, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + fn_ptr: Self::Value, + ) -> Self::Type; fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; fn reg_backend_type(&self, ty: &Reg) -> Self::Type; /// The backend type used for a rust type when it's in an SSA register. diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 90aa9188c8300..db3e3f039abe2 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -9,6 +9,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Magic.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DiagnosticHandler.h" @@ -1901,6 +1902,25 @@ extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) { Mangler().getNameWithPrefix(OS, GV, true); } +extern "C" bool +LLVMRustUpgradeIntrinsicFunction(LLVMValueRef Fn, LLVMValueRef *NewFn, + bool canUpgradeDebugIntrinsicsToRecords) { + Function *F = unwrap(Fn); + Function *NewF = nullptr; + bool CanUpgrade = + UpgradeIntrinsicFunction(F, NewF, canUpgradeDebugIntrinsicsToRecords); + *NewFn = wrap(NewF); + return CanUpgrade; +} + +extern "C" bool LLVMRustIsTargetIntrinsic(unsigned ID) { +#if LLVM_VERSION_GE(20, 1) + return Intrinsic::isTargetIntrinsic(ID); +#else + return Function::isTargetIntrinsic(ID); +#endif +} + extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) { auto *CB = unwrap(CallSite); switch (CB->getIntrinsicID()) { diff --git a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs index 8dbbb4d1713a3..111330b73efbb 100644 --- a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs +++ b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs @@ -1,6 +1,6 @@ //! This module ensures that if a function's ABI requires a particular target feature, //! that target feature is enabled both on the callee and all callers. -use rustc_abi::{BackendRepr, CanonAbi, RegKind, X86Call}; +use rustc_abi::{BackendRepr, CanonAbi, ExternAbi, RegKind, X86Call}; use rustc_hir::{CRATE_HIR_ID, HirId}; use rustc_middle::mir::{self, Location, traversal}; use rustc_middle::ty::layout::LayoutCx; @@ -152,6 +152,12 @@ fn do_check_wasm_abi<'tcx>( /// or return values for which the corresponding target feature is not enabled. fn check_instance_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) { let typing_env = ty::TypingEnv::fully_monomorphized(); + let ty = instance.ty(tcx, typing_env); + if ty.is_fn() && ty.fn_sig(tcx).abi() == ExternAbi::Unadjusted { + // We disable all checks for the unadjusted abi to allow linking to arbitrary LLVM + // intrinsics + return; + } let Ok(abi) = tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty()))) else { // An error will be reported during codegen if we cannot determine the ABI of this @@ -184,9 +190,12 @@ fn check_call_site_abi<'tcx>( caller: InstanceKind<'tcx>, loc: impl Fn() -> (Span, HirId) + Copy, ) { - if callee.fn_sig(tcx).abi().is_rustic_abi() { + let extern_abi = callee.fn_sig(tcx).abi(); + if extern_abi.is_rustic_abi() || extern_abi == ExternAbi::Unadjusted { // We directly handle the soundness of Rust ABIs -- so let's skip the majority of // call sites to avoid a perf regression. + // We disable all checks for the unadjusted abi to allow linking to arbitrary LLVM + // intrinsics return; } let typing_env = ty::TypingEnv::fully_monomorphized(); diff --git a/tests/codegen/inject-autocast.rs b/tests/codegen/inject-autocast.rs new file mode 100644 index 0000000000000..2d1cbb0d98709 --- /dev/null +++ b/tests/codegen/inject-autocast.rs @@ -0,0 +1,119 @@ +//@ compile-flags: -C opt-level=0 +//@ only-x86_64 + +#![feature(link_llvm_intrinsics, abi_unadjusted, repr_simd, simd_ffi, portable_simd, f16)] +#![crate_type = "lib"] + +use std::simd::{f32x4, i16x8, i64x2}; + +#[repr(simd)] +pub struct Tile([i8; 1024]); + +#[repr(C, packed)] +pub struct Bar(u32, i64x2, i64x2, i64x2, i64x2, i64x2, i64x2); +// CHECK: %Bar = type <{ i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> }> + +#[repr(simd)] +pub struct f16x8([f16; 8]); + +// CHECK-LABEL: @amx_autocast +#[no_mangle] +pub unsafe fn amx_autocast(m: u16, n: u16, k: u16, a: Tile, b: Tile, c: Tile) -> Tile { + extern "unadjusted" { + #[link_name = "llvm.x86.tdpbuud.internal"] + fn foo(m: u16, n: u16, k: u16, a: Tile, b: Tile, c: Tile) -> Tile; + } + + // CHECK: %3 = call x86_amx @llvm.x86.cast.vector.to.tile.v1024i8(<1024 x i8> %0) + // CHECK-NEXT: %4 = call x86_amx @llvm.x86.cast.vector.to.tile.v1024i8(<1024 x i8> %1) + // CHECK-NEXT: %5 = call x86_amx @llvm.x86.cast.vector.to.tile.v1024i8(<1024 x i8> %2) + // CHECK-NEXT: %6 = call x86_amx @llvm.x86.tdpbuud.internal(i16 %m, i16 %n, i16 %k, x86_amx %3, x86_amx %4, x86_amx %5) + // CHECK-NEXT: %7 = call <1024 x i8> @llvm.x86.cast.tile.to.vector.v1024i8(x86_amx %6) + foo(m, n, k, a, b, c) +} + +// CHECK-LABEL: @struct_with_i1_vector_autocast +#[no_mangle] +pub unsafe fn struct_with_i1_vector_autocast(a: i64x2, b: i64x2) -> (u8, u8) { + extern "unadjusted" { + #[link_name = "llvm.x86.avx512.vp2intersect.q.128"] + fn foo(a: i64x2, b: i64x2) -> (u8, u8); + } + + // CHECK: %2 = call { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64> %0, <2 x i64> %1) + // CHECK-NEXT: %3 = extractvalue { <2 x i1>, <2 x i1> } %2, 0 + // CHECK-NEXT: %4 = shufflevector <2 x i1> %3, <2 x i1> zeroinitializer, <8 x i32> + // CHECK-NEXT: %5 = bitcast <8 x i1> %4 to i8 + // CHECK-NEXT: %6 = insertvalue { i8, i8 } poison, i8 %5, 0 + // CHECK-NEXT: %7 = extractvalue { <2 x i1>, <2 x i1> } %2, 1 + // CHECK-NEXT: %8 = shufflevector <2 x i1> %7, <2 x i1> zeroinitializer, <8 x i32> + // CHECK-NEXT: %9 = bitcast <8 x i1> %8 to i8 + // CHECK-NEXT: %10 = insertvalue { i8, i8 } %6, i8 %9, 1 + foo(a, b) +} + +// CHECK-LABEL: @bf16_vector_autocast +#[no_mangle] +pub unsafe fn bf16_vector_autocast(a: f32x4) -> i16x8 { + extern "unadjusted" { + #[link_name = "llvm.x86.vcvtneps2bf16128"] + fn foo(a: f32x4) -> i16x8; + } + + // CHECK: %1 = call <8 x bfloat> @llvm.x86.vcvtneps2bf16128(<4 x float> %0) + // CHECK-NEXT: %2 = bitcast <8 x bfloat> %1 to <8 x i16> + foo(a) +} + +// CHECK-LABEL: @struct_autocast +#[no_mangle] +pub unsafe fn struct_autocast(key_metadata: u32, key: i64x2) -> Bar { + extern "unadjusted" { + #[link_name = "llvm.x86.encodekey128"] + fn foo(key_metadata: u32, key: i64x2) -> Bar; + } + + // CHECK: %1 = call { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.encodekey128(i32 %key_metadata, <2 x i64> %0) + // CHECK-NEXT: %2 = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %1, 0 + // CHECK-NEXT: %3 = insertvalue %Bar poison, i32 %2, 0 + // CHECK-NEXT: %4 = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %1, 1 + // CHECK-NEXT: %5 = insertvalue %Bar %3, <2 x i64> %4, 1 + // CHECK-NEXT: %6 = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %1, 2 + // CHECK-NEXT: %7 = insertvalue %Bar %5, <2 x i64> %6, 2 + // CHECK-NEXT: %8 = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %1, 3 + // CHECK-NEXT: %9 = insertvalue %Bar %7, <2 x i64> %8, 3 + // CHECK-NEXT: %10 = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %1, 4 + // CHECK-NEXT: %11 = insertvalue %Bar %9, <2 x i64> %10, 4 + // CHECK-NEXT: %12 = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %1, 5 + // CHECK-NEXT: %13 = insertvalue %Bar %11, <2 x i64> %12, 5 + // CHECK-NEXT: %14 = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %1, 6 + // CHECK-NEXT: %15 = insertvalue %Bar %13, <2 x i64> %14, 6 + foo(key_metadata, key) +} + +// CHECK-LABEL: @i1_vector_autocast +#[no_mangle] +pub unsafe fn i1_vector_autocast(a: f16x8) -> u8 { + extern "unadjusted" { + #[link_name = "llvm.x86.avx512fp16.fpclass.ph.128"] + fn foo(a: f16x8, b: i32) -> u8; + } + + // CHECK: %1 = call <8 x i1> @llvm.x86.avx512fp16.fpclass.ph.128(<8 x half> %0, i32 1) + // CHECK-NEXT: %_0 = bitcast <8 x i1> %1 to i8 + foo(a, 1) +} + +// CHECK: declare x86_amx @llvm.x86.tdpbuud.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx) + +// CHECK: declare x86_amx @llvm.x86.cast.vector.to.tile.v1024i8(<1024 x i8>) + +// CHECK: declare <1024 x i8> @llvm.x86.cast.tile.to.vector.v1024i8(x86_amx) + +// CHECK: declare { <2 x i1>, <2 x i1> } @llvm.x86.avx512.vp2intersect.q.128(<2 x i64>, <2 x i64>) + +// CHECK: declare <8 x bfloat> @llvm.x86.vcvtneps2bf16128(<4 x float>) + +// CHECK: declare { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.encodekey128(i32, <2 x i64>) + +// CHECK: declare <8 x i1> @llvm.x86.avx512fp16.fpclass.ph.128(<8 x half>, i32 immarg) diff --git a/tests/run-make/simd-ffi/simd.rs b/tests/run-make/simd-ffi/simd.rs index 9ea8eb8cf8831..2a2032f218e75 100644 --- a/tests/run-make/simd-ffi/simd.rs +++ b/tests/run-make/simd-ffi/simd.rs @@ -35,7 +35,7 @@ extern "C" { fn integer(a: i32x4, b: i32x4) -> i32x4; // vmaxq_s32 #[cfg(target_arch = "aarch64")] - #[link_name = "llvm.aarch64.neon.maxs.v4i32"] + #[link_name = "llvm.aarch64.neon.smax.v4i32"] fn integer(a: i32x4, b: i32x4) -> i32x4; // Use a generic LLVM intrinsic to do type checking on other platforms diff --git a/tests/ui/codegen/deprecated-llvm-intrinsic.rs b/tests/ui/codegen/deprecated-llvm-intrinsic.rs new file mode 100644 index 0000000000000..f546c69ce9cc8 --- /dev/null +++ b/tests/ui/codegen/deprecated-llvm-intrinsic.rs @@ -0,0 +1,27 @@ +//@ add-core-stubs +//@ build-pass +//@ ignore-pass +//@ compile-flags: --target aarch64-unknown-linux-gnu +//@ needs-llvm-components: aarch64 +#![feature(no_core, lang_items, link_llvm_intrinsics, abi_unadjusted, repr_simd, simd_ffi)] +#![no_std] +#![no_core] +#![allow(internal_features, non_camel_case_types, improper_ctypes)] +#![crate_type = "lib"] + +extern crate minicore; +use minicore::*; + +#[repr(simd)] +pub struct i8x8([i8; 8]); + +extern "unadjusted" { + #[link_name = "llvm.aarch64.neon.rbit.v8i8"] + fn foo(a: i8x8) -> i8x8; + //~^ NOTE: Using deprecated intrinsic `llvm.aarch64.neon.rbit.v8i8`, `llvm.bitreverse.v8i8` can be used instead +} + +#[target_feature(enable = "neon")] +pub unsafe fn bar(a: i8x8) -> i8x8 { + foo(a) +} diff --git a/tests/ui/codegen/deprecated-llvm-intrinsic.stderr b/tests/ui/codegen/deprecated-llvm-intrinsic.stderr new file mode 100644 index 0000000000000..b0d578e4f918c --- /dev/null +++ b/tests/ui/codegen/deprecated-llvm-intrinsic.stderr @@ -0,0 +1,6 @@ +note: Using deprecated intrinsic `llvm.aarch64.neon.rbit.v8i8`, `llvm.bitreverse.v8i8` can be used instead + --> $DIR/deprecated-llvm-intrinsic.rs:20:5 + | +LL | fn foo(a: i8x8) -> i8x8; + | ^^^^^^^^^^^^^^^^^^^^^^^^ + diff --git a/tests/ui/codegen/incorrect-arch-intrinsic.rs b/tests/ui/codegen/incorrect-arch-intrinsic.rs new file mode 100644 index 0000000000000..22743b80aa62d --- /dev/null +++ b/tests/ui/codegen/incorrect-arch-intrinsic.rs @@ -0,0 +1,17 @@ +//@ build-fail +//@ ignore-s390x +//@ normalize-stderr: "target arch `(.*)`" -> "target arch `TARGET_ARCH`" + +#![feature(link_llvm_intrinsics, abi_unadjusted)] + +extern "unadjusted" { + #[link_name = "llvm.s390.sfpc"] + fn foo(a: i32); + //~^ ERROR: Intrinsic `llvm.s390.sfpc` cannot be used with target arch +} + +pub fn main() { + unsafe { + foo(0); + } +} diff --git a/tests/ui/codegen/incorrect-arch-intrinsic.stderr b/tests/ui/codegen/incorrect-arch-intrinsic.stderr new file mode 100644 index 0000000000000..c4ecf2ed11724 --- /dev/null +++ b/tests/ui/codegen/incorrect-arch-intrinsic.stderr @@ -0,0 +1,8 @@ +error: Intrinsic `llvm.s390.sfpc` cannot be used with target arch `TARGET_ARCH` + --> $DIR/incorrect-arch-intrinsic.rs:9:5 + | +LL | fn foo(a: i32); + | ^^^^^^^^^^^^^^^ + +error: aborting due to 1 previous error + diff --git a/tests/ui/codegen/incorrect-llvm-intrinsic-signature.rs b/tests/ui/codegen/incorrect-llvm-intrinsic-signature.rs new file mode 100644 index 0000000000000..f272fb1d19a80 --- /dev/null +++ b/tests/ui/codegen/incorrect-llvm-intrinsic-signature.rs @@ -0,0 +1,13 @@ +//@ build-fail + +#![feature(link_llvm_intrinsics, abi_unadjusted)] + +extern "unadjusted" { + #[link_name = "llvm.assume"] + fn foo(); + //~^ ERROR: Intrinsic signature mismatch for `llvm.assume`: expected signature `void (i1)`, found `void ()` +} + +pub fn main() { + unsafe { foo() } +} diff --git a/tests/ui/codegen/incorrect-llvm-intrinsic-signature.stderr b/tests/ui/codegen/incorrect-llvm-intrinsic-signature.stderr new file mode 100644 index 0000000000000..e443fb7d48534 --- /dev/null +++ b/tests/ui/codegen/incorrect-llvm-intrinsic-signature.stderr @@ -0,0 +1,8 @@ +error: Intrinsic signature mismatch for `llvm.assume`: expected signature `void (i1)`, found `void ()` + --> $DIR/incorrect-llvm-intrinsic-signature.rs:7:5 + | +LL | fn foo(); + | ^^^^^^^^^ + +error: aborting due to 1 previous error + diff --git a/tests/ui/codegen/invalid-llvm-intrinsic.rs b/tests/ui/codegen/invalid-llvm-intrinsic.rs new file mode 100644 index 0000000000000..01edb114d0086 --- /dev/null +++ b/tests/ui/codegen/invalid-llvm-intrinsic.rs @@ -0,0 +1,13 @@ +//@ build-fail + +#![feature(link_llvm_intrinsics, abi_unadjusted)] + +extern "unadjusted" { + #[link_name = "llvm.abcde"] + fn foo(); + //~^ ERROR: Invalid LLVM Intrinsic `llvm.abcde` +} + +pub fn main() { + unsafe { foo() } +} diff --git a/tests/ui/codegen/invalid-llvm-intrinsic.stderr b/tests/ui/codegen/invalid-llvm-intrinsic.stderr new file mode 100644 index 0000000000000..39a6ca52895e8 --- /dev/null +++ b/tests/ui/codegen/invalid-llvm-intrinsic.stderr @@ -0,0 +1,8 @@ +error: Invalid LLVM Intrinsic `llvm.abcde` + --> $DIR/invalid-llvm-intrinsic.rs:7:5 + | +LL | fn foo(); + | ^^^^^^^^^ + +error: aborting due to 1 previous error +