diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 5503f7689304b..91fa7002cf014 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -197,6 +197,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &[ptr, args[1].immediate()], ) } + sym::addrspace_ptr_cast | sym::addrspace_ptr_from_ptr | sym::addrspace_ptr_to_ptr => { + self.pointercast(args[0].immediate(), result.layout.immediate_llvm_type(self.cx)) + } + sym::addrspace_ptr_to_addr => { + self.ptrtoint(args[0].immediate(), result.layout.immediate_llvm_type(self.cx)) + } sym::autodiff => { codegen_autodiff(self, tcx, instance, args, result); return Ok(()); diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index f4fae40d8828f..4501abcee5c06 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -186,7 +186,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } value } - sym::arith_offset => { + sym::arith_offset | sym::addrspace_ptr_arith_offset => { let ty = fn_args.type_at(0); let layout = bx.layout_of(ty); let ptr = args[0].immediate(); diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 4ac3e4e83e80a..e66e60938750c 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -186,6 +186,7 @@ language_item_table! { PointeeTrait, sym::pointee_trait, pointee_trait, Target::Trait, GenericRequirement::None; Metadata, sym::metadata_type, metadata_type, Target::AssocTy, GenericRequirement::None; DynMetadata, sym::dyn_metadata, dyn_metadata, Target::Struct, GenericRequirement::None; + AddrspacePtr, sym::addrspace_ptr_type, addrspace_ptr_type, Target::Struct, GenericRequirement::Exact(2); Freeze, sym::freeze, freeze_trait, Target::Trait, GenericRequirement::Exact(0); UnsafeUnpin, sym::unsafe_unpin, unsafe_unpin_trait, Target::Trait, GenericRequirement::Exact(0); diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 4e8333f678b66..ecf2b73743d47 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -4,7 +4,7 @@ use rustc_abi::ExternAbi; use rustc_errors::DiagMessage; use rustc_hir::{self as hir, LangItem}; use rustc_middle::traits::{ObligationCause, ObligationCauseCode}; -use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_middle::ty::{self, Const, Ty, TyCtxt}; use rustc_span::def_id::LocalDefId; use rustc_span::{Span, Symbol, sym}; @@ -239,8 +239,8 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi /// Remember to add all intrinsics here, in `compiler/rustc_codegen_llvm/src/intrinsic.rs`, /// and in `library/core/src/intrinsics.rs`. -pub(crate) fn check_intrinsic_type( - tcx: TyCtxt<'_>, +pub(crate) fn check_intrinsic_type<'tcx>( + tcx: TyCtxt<'tcx>, intrinsic_id: LocalDefId, span: Span, intrinsic_name: Symbol, @@ -255,6 +255,15 @@ pub(crate) fn check_intrinsic_type( Ty::new_error_with_message(tcx, span, "expected param") } }; + let const_param = |n| { + if let &ty::GenericParamDef { name, kind: ty::GenericParamDefKind::Const { .. }, .. } = + generics.param_at(n as usize, tcx) + { + Const::new_param(tcx, ty::ParamConst::new(n, name)) + } else { + Const::new_error_with_message(tcx, span, "expected const param") + } + }; let bound_vars = tcx.mk_bound_variable_kinds(&[ ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon), @@ -280,9 +289,49 @@ pub(crate) fn check_intrinsic_type( (Ty::new_ref(tcx, env_region, va_list_ty, mutbl), va_list_ty) }; + let mk_u32_const = + |i: u32| Const::new_value(tcx, ty::ValTree::from_scalar_int(tcx, i.into()), tcx.types.u32); + let mk_addrspace_ptr = |t: Ty<'tcx>, addrspace: Const<'tcx>| { + tcx.type_of(tcx.lang_items().addrspace_ptr_type().unwrap()) + .instantiate(tcx, &[t.into(), addrspace.into()]) + }; + let safety = intrinsic_operation_unsafety(tcx, intrinsic_id); let n_lts = 0; let (n_tps, n_cts, inputs, output) = match intrinsic_name { + sym::addrspace_ptr_arith_offset | sym::addrspace_ptr_offset => ( + 1, + 1, + vec![mk_addrspace_ptr(param(0), const_param(1)), tcx.types.isize], + mk_addrspace_ptr(param(0), const_param(1)), + ), + sym::addrspace_ptr_cast => ( + 2, + 2, + vec![mk_addrspace_ptr(param(0), const_param(2))], + mk_addrspace_ptr(param(1), const_param(3)), + ), + sym::addrspace_ptr_to_addr => { + (1, 1, vec![mk_addrspace_ptr(tcx.types.unit, const_param(1))], param(0)) + } + sym::addrspace_ptr_from_ptr => ( + 1, + 0, + vec![Ty::new_mut_ptr(tcx, param(0).into())], + mk_addrspace_ptr(param(0), mk_u32_const(0)), + ), + sym::addrspace_ptr_to_ptr => ( + 1, + 0, + vec![mk_addrspace_ptr(param(0), mk_u32_const(0))], + Ty::new_mut_ptr(tcx, param(0)), + ), + sym::addrspace_ptr_read_via_copy => { + (1, 1, vec![mk_addrspace_ptr(param(0), const_param(1))], param(0)) + } + sym::addrspace_ptr_write_via_move => { + (1, 1, vec![mk_addrspace_ptr(param(0), const_param(1)), param(0)], tcx.types.unit) + } sym::autodiff => (4, 0, vec![param(0), param(1), param(2)], param(3)), sym::abort => (0, 0, vec![], tcx.types.never), sym::unreachable => (0, 0, vec![], tcx.types.never), diff --git a/compiler/rustc_hir_typeck/src/place_op.rs b/compiler/rustc_hir_typeck/src/place_op.rs index a48db2cc855c0..40e52914aec37 100644 --- a/compiler/rustc_hir_typeck/src/place_op.rs +++ b/compiler/rustc_hir_typeck/src/place_op.rs @@ -23,7 +23,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { oprnd_expr: &'tcx hir::Expr<'tcx>, oprnd_ty: Ty<'tcx>, ) -> Option> { - if let Some(ty) = oprnd_ty.builtin_deref(true) { + // AddrspacePtr deref is used internally but it should not be accessed from the Rust, + // so exclude it here. + if let Some(ty) = oprnd_ty.builtin_deref(true) + && !oprnd_ty.is_addrspace_ptr() + { return Some(ty); } diff --git a/compiler/rustc_lint/src/types/improper_ctypes.rs b/compiler/rustc_lint/src/types/improper_ctypes.rs index 38094c67c34a0..cb5d2d405d000 100644 --- a/compiler/rustc_lint/src/types/improper_ctypes.rs +++ b/compiler/rustc_lint/src/types/improper_ctypes.rs @@ -468,6 +468,9 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> { if def.is_phantom_data() { return FfiPhantom(ty); } + if def.is_addrspace_ptr() { + return FfiSafe; + } match def.adt_kind() { AdtKind::Struct | AdtKind::Union => { if let Some(sym::cstring_type | sym::cstr_type) = diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs index 510c546f82a4e..e1cf397288048 100644 --- a/compiler/rustc_middle/src/ty/adt.rs +++ b/compiler/rustc_middle/src/ty/adt.rs @@ -59,6 +59,8 @@ bitflags::bitflags! { const IS_PIN = 1 << 11; /// Indicates whether the type is `#[pin_project]`. const IS_PIN_PROJECT = 1 << 12; + /// Indicates whether the type is `AddrspacePtr`. + const IS_ADDRSPACE_PTR = 1 << 13; } } rustc_data_structures::external_bitflags_debug! { AdtFlags } @@ -324,6 +326,9 @@ impl AdtDefData { if tcx.is_lang_item(did, LangItem::Pin) { flags |= AdtFlags::IS_PIN; } + if tcx.is_lang_item(did, LangItem::AddrspacePtr) { + flags |= AdtFlags::IS_ADDRSPACE_PTR; + } AdtDefData { did, variants, flags, repr } } @@ -445,6 +450,12 @@ impl<'tcx> AdtDef<'tcx> { self.flags().contains(AdtFlags::IS_PIN) } + /// Returns `true` if this is `AddrspacePtr`. + #[inline] + pub fn is_addrspace_ptr(self) -> bool { + self.flags().contains(AdtFlags::IS_ADDRSPACE_PTR) + } + /// Returns `true` is this is `#[pin_v2]` for the purposes /// of structural pinning. #[inline] diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index c3834607e9236..669de60238508 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -1339,6 +1339,14 @@ impl<'tcx> Ty<'tcx> { } } + #[inline] + pub fn is_addrspace_ptr(self) -> bool { + match self.kind() { + Adt(def, _) => def.is_addrspace_ptr(), + _ => false, + } + } + /// Tests whether this is a Box definitely using the global allocator. /// /// If the allocator is still generic, the answer is `false`, but it may @@ -1366,6 +1374,13 @@ impl<'tcx> Ty<'tcx> { } } + pub fn addrspace_ptr_ty(self) -> Option> { + match self.kind() { + Adt(def, args) if def.is_addrspace_ptr() => Some(args.type_at(0)), + _ => None, + } + } + pub fn pinned_ty(self) -> Option> { match self.kind() { Adt(def, args) if def.is_pin() => Some(args.type_at(0)), @@ -1566,6 +1581,11 @@ impl<'tcx> Ty<'tcx> { pub fn builtin_deref(self, explicit: bool) -> Option> { match *self.kind() { _ if let Some(boxed) = self.boxed_ty() => Some(boxed), + _ if let Some(ty) = self.addrspace_ptr_ty() + && explicit => + { + Some(ty) + } Ref(_, ty, _) => Some(ty), RawPtr(ty, _) if explicit => Some(ty), _ => None, diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index dcee54c371080..613ea1e516593 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -145,7 +145,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { )); terminator.kind = TerminatorKind::Goto { target }; } - sym::read_via_copy => { + sym::read_via_copy | sym::addrspace_ptr_read_via_copy => { let Ok([arg]) = take_array(args) else { span_bug!(terminator.source_info.span, "Wrong number of arguments"); }; @@ -177,7 +177,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { Some(target) => TerminatorKind::Goto { target }, } } - sym::write_via_move => { + sym::write_via_move | sym::addrspace_ptr_write_via_move => { let target = target.unwrap(); let Ok([ptr, val]) = take_array(args) else { span_bug!( @@ -220,7 +220,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { )); terminator.kind = TerminatorKind::Goto { target }; } - sym::offset => { + sym::offset | sym::addrspace_ptr_offset => { let target = target.unwrap(); let Ok([ptr, delta]) = take_array(args) else { span_bug!( diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 85e340c0a02ab..ac8a60aabaad7 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -543,6 +543,7 @@ fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) - match self_ty.kind() { ty::FnDef(..) | ty::FnPtr(..) => builder.copy_shim(), + ty::Adt(def, _) if def.is_addrspace_ptr() => builder.copy_shim(), ty::Closure(_, args) => builder.tuple_like_shim(dest, src, args.as_closure().upvar_tys()), ty::CoroutineClosure(_, args) => { builder.tuple_like_shim(dest, src, args.as_coroutine_closure().upvar_tys()) diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 1afdb4639a0ce..521c83cdd74c3 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -1178,7 +1178,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { match op { Offset => { - check_kinds!(a, "Cannot offset non-pointer type {:?}", ty::RawPtr(..)); + if !a.is_addrspace_ptr() { + check_kinds!(a, "Cannot offset non-pointer type {:?}", ty::RawPtr(..)); + } if b != self.tcx.types.isize && b != self.tcx.types.usize { self.fail(location, format!("Cannot offset by non-isize type {b}")); } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 0e30abccb62b3..04c638ab4a0a8 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -428,6 +428,15 @@ symbols! { add_assign, add_with_overflow, address, + addrspace_ptr_arith_offset, + addrspace_ptr_cast, + addrspace_ptr_from_ptr, + addrspace_ptr_offset, + addrspace_ptr_read_via_copy, + addrspace_ptr_to_addr, + addrspace_ptr_to_ptr, + addrspace_ptr_type, + addrspace_ptr_write_via_move, adt_const_params, advanced_slice_patterns, adx_target_feature, diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs index d0833f0308350..0474450e6ea49 100644 --- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs +++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs @@ -1207,6 +1207,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { candidates.vec.push(SizedCandidate); } + ty::Adt(def, _) if def.is_addrspace_ptr() => { + candidates.vec.push(BuiltinCandidate); + } + // Fallback to whatever user-defined impls or param-env clauses exist in this case. ty::Adt(..) | ty::Alias(..) | ty::Param(..) | ty::Placeholder(..) => {} diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs index ea903bac9d617..72b05939d443a 100644 --- a/compiler/rustc_trait_selection/src/traits/select/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs @@ -2256,6 +2256,8 @@ impl<'tcx> SelectionContext<'_, 'tcx> { ty::Binder::dummy(args.as_coroutine_closure().upvar_tys().to_vec()) } + ty::Adt(def, args) if def.is_addrspace_ptr() => ty::Binder::dummy(vec![]), + ty::Foreign(..) | ty::Str | ty::Slice(_) diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs index 23bbd9ca6d639..82cebdcd3bf04 100644 --- a/compiler/rustc_ty_utils/src/instance.rs +++ b/compiler/rustc_ty_utils/src/instance.rs @@ -268,6 +268,7 @@ fn resolve_associated_item<'tcx>( | ty::Closure(..) | ty::CoroutineClosure(..) | ty::Tuple(..) => {} + ty::Adt(def, _) if def.is_addrspace_ptr() => {} _ => return Ok(None), }; diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 62f3667ad7f4f..effd6defbbbcf 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -643,6 +643,21 @@ fn layout_of_uncached<'tcx>( // ADTs. ty::Adt(def, args) => { + if def.is_addrspace_ptr() { + // Implement the AddrspacePtr lang item struct as a pointer into the + // address space from the const generic argument. + let pointee = args.type_at(0); + let addrspace = extract_const_value(cx, ty, args.const_at(1))? + .try_to_bits(tcx, ty::TypingEnv::fully_monomorphized()) + .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))? + as u32; + let data_ptr = scalar_unit(Pointer(AddressSpace(addrspace))); + if !pointee.is_sized(tcx, cx.typing_env) { + panic!("AddrspacePtr pointee must be sized"); + } + return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr))); + } + // Cache the field layouts. let variants = def .variants() diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 2179b451c375e..1f58207ae7261 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -56,6 +56,7 @@ use crate::ffi::va_list::{VaArgSafe, VaList}; use crate::marker::{ConstParamTy, Destruct, DiscriminantKind, PointeeSized, Tuple}; +use crate::ptr::AddrspacePtr; use crate::{mem, ptr}; mod bounds; @@ -910,6 +911,138 @@ pub const unsafe fn offset(dst: Ptr, offset: D #[rustc_intrinsic] pub const unsafe fn arith_offset(dst: *const T, offset: isize) -> *const T; +/// Casts a pointer to a pointer in a different address space. +/// +/// A pointer into one address space can sometimes be casted to a pointer into a different address space. +/// The target defines for which combination of address spaces this is possible. +/// +/// This intrinsic also allows casting the pointed-to type of the pointer. +/// +/// # Safety +/// +/// The cast must be supported by the target, casting between address spaces that do not support it +/// is undefined behavior and can fail to compile. +#[must_use = "returns a new pointer rather than modifying its argument"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_cast< + T: 'static, + U: 'static, + const SOURCE_ADDRSPACE: u32, + const TARGET_ADDRSPACE: u32, +>( + ptr: AddrspacePtr, +) -> AddrspacePtr; + +/// Converts a raw pointer into a pointer with an explicit address space. +/// +/// If the targets raw pointer address space is the generic address space, this is a no-op, +/// just casting the type. +/// +/// # Safety +/// +/// If the targets raw pointer address space matches the generic address space, this function +/// is a no-op and is always safe to call. +/// Otherwise, if the address spaces do not match, the safety conditions of [`addrspace_ptr_cast`] apply. +#[must_use = "returns a new pointer rather than modifying its argument"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_from_ptr( + ptr: *mut T, +) -> AddrspacePtr; + +/// Converts a pointer with an explicit address space into a raw pointer. +/// +/// If the targets raw pointer address space is the generic address space, this is a no-op, +/// just casting the type. +/// +/// # Safety +/// +/// If the targets raw pointer address space matches the generic address space, this function +/// is a no-op and is always safe to call. +/// Otherwise, if the address spaces do not match, the safety conditions of [`addrspace_ptr_cast`] apply. +#[must_use = "returns a new pointer rather than modifying its argument"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_to_ptr( + ptr: AddrspacePtr, +) -> *mut T; + +/// Gets the "address" portion of a pointer. +/// +/// The return type `T` must be an integer of the same size as the pointer. +/// The size depends on the target and can differ between address spaces. +/// This does not expose the provenance of the pointer. +/// +/// See also [`pointer::addr`] for the raw pointer equivalent. +/// +/// # Safety +/// +/// Not all address spaces have pointers that can be converted to integers. +/// Calling this intrinsic for non-integral pointers results in undefined behavior. +/// +/// The returned type must be an integer matching the size of the pointer, otherwise calling +/// this intrinsic results in undefined behavior. +#[must_use = "returns the address and does nothing unless used"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_to_addr(ptr: AddrspacePtr<(), ADDRSPACE>) +-> T; + +/// Adds a signed offset to a pointer. +/// +/// `count` is in units of `T`. +/// +/// See [`pointer::offset`] for details and safety concerns. +#[must_use = "returns a new pointer rather than modifying its argument"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_offset( + ptr: AddrspacePtr, + count: isize, +) -> AddrspacePtr; + +/// Adds a signed offset to a pointer using wrapping arithmetic. +/// +/// `count` is in units of `T`. +/// +/// See [`pointer::wrapping_offset`] for details and safety concerns. +#[must_use = "returns a new pointer rather than modifying its argument"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_arith_offset( + ptr: AddrspacePtr, + count: isize, +) -> AddrspacePtr; + +/// Reads the value from `self` without moving it. This leaves the memory in `self` unchanged. +/// +/// See [`ptr::read`] for safety concerns and examples. +#[must_use = "returns the read value and does nothing unless used"] +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_read_via_copy( + ptr: AddrspacePtr, +) -> T; + +/// Overwrites a memory location with the given value without reading or dropping the old value. +/// +/// See [`ptr::write`] for safety concerns and examples. +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[rustc_intrinsic] +#[rustc_nounwind] +pub unsafe fn addrspace_ptr_write_via_move( + ptr: AddrspacePtr, + value: T, +); + /// Projects to the `index`-th element of `slice_ptr`, as the same kind of pointer /// as the slice was provided -- so `&mut [T] → &mut T`, `&[T] → &T`, /// `*mut [T] → *mut T`, or `*const [T] → *const T` -- without a bounds check. diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index 29fe77390a16b..7fab9849064f0 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -403,7 +403,7 @@ use crate::cmp::Ordering; use crate::intrinsics::const_eval_select; -use crate::marker::{Destruct, FnPtr, PointeeSized}; +use crate::marker::{Destruct, FnPtr, PhantomData, PointeeSized}; use crate::mem::{self, MaybeUninit, SizedTypeProperties}; use crate::num::NonZero; use crate::{fmt, hash, intrinsics, ub_checks}; @@ -427,6 +427,237 @@ pub use unique::Unique; mod const_ptr; mod mut_ptr; +/// The list of address spaces. +/// +/// See [`AddrspacePtr`] for using pointers with an explicit associated address space. +/// +/// This module lists the address spaces for the current target. +#[unstable(feature = "ptr_addrspace", issue = "none")] +pub mod addrspace { + /// The default address space. + /// + /// On most targets, a raw pointer (`*mut/const T`) implicitly points into the generic address space. + #[unstable(feature = "ptr_addrspace", issue = "none")] + pub const GENERIC: u32 = 0; + + /// The address space corresponding to VRAM on GPUs. + /// + /// It is readable and writable from the GPU and CPU. + /// Pointers into this address space can be casted to and from the generic address space. + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[cfg(any(doc, target_arch = "amdgpu", target_arch = "nvptx64"))] + #[doc(cfg(any(target_arch = "amdgpu", target_arch = "nvptx64")))] + pub const GLOBAL: u32 = 1; + + /// The address space shared between threads within the same workgroup on GPUs. + /// + /// It is readable and writable from the GPU. + /// Pointers into this address space can be casted to and from the generic address space. + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[cfg(any(doc, target_arch = "amdgpu", target_arch = "nvptx64"))] + #[doc(cfg(any(target_arch = "amdgpu", target_arch = "nvptx64")))] + pub const WORKGROUP: u32 = 3; + + /// The address space for constant memory on GPUs. + /// + /// It is readable from the GPU but not writable. + /// Pointers into this address space can be casted to and from the generic address space. + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[cfg(any(doc, target_arch = "amdgpu", target_arch = "nvptx64"))] + #[doc(cfg(any(target_arch = "amdgpu", target_arch = "nvptx64")))] + pub const CONST: u32 = 4; +} + +/// A pointer into a target-specific address space. +/// +/// An address space is a special, target-specific memory region. +/// Each pointer has an associated address space. If no address space is explicitly mentioned, it +/// is usually the generic address space. When accessing low-level hardware capabilities or +/// intrinsics, it is sometimes necessary to explicitly specify a different address space. +/// For these cases, `AddrspacePointer` can be used. +/// +/// See also the [pointer](pointer) primitive types and the [`addrspace`] module. +#[unstable(feature = "ptr_addrspace", issue = "none")] +#[lang = "addrspace_ptr_type"] +#[allow(missing_debug_implementations)] +pub struct AddrspacePtr { + // Struct implementation is replaced by the compiler. + // This field is here for using the generic argument but cannot be set or accessed in any way. + do_not_use: PhantomData<*const T>, +} + +impl AddrspacePtr { + /// Gets the "address" portion of a pointer. + /// + /// The return type `U` must be an integer of the same size as the pointer. + /// The size depends on the target and can differ between address spaces. + /// This does not expose the provenance of the pointer. + /// + /// See also [`pointer::addr`] for the raw pointer equivalent. + /// + /// # Safety + /// + /// Not all address spaces have pointers that can be converted to integers. + /// Calling this intrinsic for non-integral pointers results in undefined behavior. + /// + /// The returned type must be an integer matching the size of the pointer, otherwise calling + /// this intrinsic results in undefined behavior. + #[must_use = "returns the address and does nothing unless used"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn addr(self) -> U { + // SAFETY: the safety contract for `addrspace_ptr_to_addr` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_to_addr(self.cast::<()>()) } + } + + /// Cast to a pointer of different type. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub fn cast(self) -> AddrspacePtr { + // SAFETY: Only the pointed-to type is converted, making this a no-op just converting the type. + unsafe { crate::intrinsics::addrspace_ptr_cast(self) } + } + + /// Casts a pointer to a pointer in a different address space. + /// + /// A pointer into one address space can sometimes be casted to a pointer into a different address space. + /// The target defines for which combination of address spaces this is possible. + /// + /// # Safety + /// + /// The cast must be supported by the target, casting between address spaces that do not support it + /// is undefined behavior and can fail to compile. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn cast_addrspace( + self, + ) -> AddrspacePtr { + // SAFETY: the safety contract for `addrspace_ptr_cast` must be upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_cast(self) } + } + + /// Adds a signed offset to a pointer. + /// + /// `count` is in units of `T`. + /// + /// See [`pointer::offset`] for details and safety concerns. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + #[track_caller] + pub unsafe fn offset(self, count: isize) -> Self { + // We cannot convert the addrspace ptr to an integer as it could be a non-integra pointer, + // so just check that the multiplication does not overflow. + #[inline] + #[rustc_allow_const_fn_unstable(const_eval_select)] + const fn runtime_offset_nowrap(count: isize, size: usize) -> bool { + // We can use const_eval_select here because this is only for UB checks. + const_eval_select!( + @capture { count: isize, size: usize } -> bool: + if const { + true + } else { + // `size` is the size of a Rust type, so we know that + // `size <= isize::MAX` and thus `as` cast here is not lossy. + count.checked_mul(size as isize).is_some() + } + ) + } + + ub_checks::assert_unsafe_precondition!( + check_language_ub, + "AddrspacePtr::offset requires the address calculation to not overflow", + ( + count: isize = count, + size: usize = size_of::(), + ) => runtime_offset_nowrap(count, size) + ); + + // SAFETY: the safety contract for `addrspace_ptr_offset` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_offset(self, count) } + } + + /// Adds a signed offset to a pointer using wrapping arithmetic. + /// + /// `count` is in units of `T`. + /// + /// See [`pointer::wrapping_offset`] for details and safety concerns. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub fn wrapping_offset(self, count: isize) -> Self { + // SAFETY: the `addrspace_ptr_arith_offset` intrinsic has no prerequisites to be called. + unsafe { crate::intrinsics::addrspace_ptr_arith_offset(self, count) } + } + + /// Reads the value from `self` without moving it. This leaves the memory in `self` unchanged. + /// + /// See [`core::ptr::read`] for safety concerns and examples. + #[must_use = "returns the read value and does nothing unless used"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn read(self) -> T { + // SAFETY: the safety contract for `addrspace_ptr_read_via_copy` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_read_via_copy(self) } + } + + /// Overwrites a memory location with the given value without reading or dropping the old value. + /// + /// See [`core::ptr::write`] for safety concerns and examples. + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn write(self, value: T) { + // SAFETY: the safety contract for `addrspace_ptr_write_via_move` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_write_via_move(self, value) } + } +} + +impl AddrspacePtr { + /// Converts a raw pointer into a pointer with an explicit address space. + /// + /// If the targets raw pointer address space is the generic address space, this is a no-op, + /// just casting the type. + /// + /// # Safety + /// + /// If the targets raw pointer address space matches the generic address space, this function + /// is a no-op and is always safe to call. + /// Otherwise, if the address spaces do not match, the safety conditions of [`AddrspacePtr::cast_addrspace`] apply. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn from_ptr(ptr: *mut T) -> Self { + // SAFETY: the safety contract for `addrspace_ptr_from_ptr` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_from_ptr(ptr) } + } + + /// Converts a pointer with an explicit address space into a raw pointer. + /// + /// If the targets raw pointer address space is the generic address space, this is a no-op, + /// just casting the type. + /// + /// # Safety + /// + /// If the targets raw pointer address space matches the generic address space, this function + /// is a no-op and is always safe to call. + /// Otherwise, if the address spaces do not match, the safety conditions of [`AddrspacePtr::cast_addrspace`] apply. + #[must_use = "returns a new pointer rather than modifying its argument"] + #[unstable(feature = "ptr_addrspace", issue = "none")] + #[inline(always)] + pub unsafe fn as_ptr(self) -> *mut T { + // SAFETY: the safety contract for `addrspace_ptr_to_ptr` must be + // upheld by the caller. + unsafe { crate::intrinsics::addrspace_ptr_to_ptr(self) } + } +} + // Some functions are defined here because they accidentally got made // available in this module on stable. See . // (`transmute` also falls into this category, but it cannot be wrapped due to the diff --git a/tests/auxiliary/minicore.rs b/tests/auxiliary/minicore.rs index 288a5b50dd5ef..bfc4db6b6cf1c 100644 --- a/tests/auxiliary/minicore.rs +++ b/tests/auxiliary/minicore.rs @@ -293,3 +293,4 @@ pub enum SimdAlign { } impl ConstParamTy_ for SimdAlign {} +impl ConstParamTy_ for u32 {} diff --git a/tests/codegen-llvm/addrspace-ptr-basic.rs b/tests/codegen-llvm/addrspace-ptr-basic.rs new file mode 100644 index 0000000000000..a80053b0c909d --- /dev/null +++ b/tests/codegen-llvm/addrspace-ptr-basic.rs @@ -0,0 +1,244 @@ +//! Checks the basic usage pointers into the non-generic address space. + +//@ add-minicore +//@ compile-flags: --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900 +//@ needs-llvm-components: amdgpu + +#![crate_type = "lib"] +#![no_core] +#![feature(abi_unadjusted, intrinsics, lang_items, link_llvm_intrinsics, no_core, rustc_attrs)] + +extern crate minicore; +use minicore::*; + +#[rustc_intrinsic] +const unsafe fn size_of_val(x: *const T) -> usize; + +mod addrspace { + pub const GENERIC: u32 = 0; + pub const WORKGROUP: u32 = 3; + pub const CONST: u32 = 4; +} + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_cast< + T: 'static, + U: 'static, + const SOURCE_ADDRSPACE: u32, + const TARGET_ADDRSPACE: u32, +>( + ptr: AddrspacePtr, +) -> AddrspacePtr; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_from_ptr( + ptr: *mut T, +) -> AddrspacePtr; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_to_ptr(ptr: AddrspacePtr) -> *mut T; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_to_addr(ptr: AddrspacePtr<(), ADDRSPACE>) -> T; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_offset( + ptr: AddrspacePtr, + count: isize, +) -> AddrspacePtr; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_arith_offset( + ptr: AddrspacePtr, + count: isize, +) -> AddrspacePtr; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_read_via_copy( + ptr: AddrspacePtr, +) -> T; + +#[rustc_intrinsic] +#[rustc_nounwind] +unsafe fn addrspace_ptr_write_via_move( + ptr: AddrspacePtr, + value: T, +); + +#[repr(C)] +struct DispatchPacket { + header: u16, +} + +extern "unadjusted" { + #[link_name = "llvm.amdgcn.dispatch.ptr"] + fn dispatch_ptr() -> AddrspacePtr; +} + +#[lang = "addrspace_ptr_type"] +pub struct AddrspacePtr { + // Struct implementation is replaced by compiler. + // This field is here for using the generic arguments but cannot be set or used in any way. + do_not_use: PhantomData<*const T>, +} + +impl AddrspacePtr { + unsafe fn addr(self) -> U { + unsafe { addrspace_ptr_to_addr(self.cast::<()>()) } + } + + unsafe fn cast(self) -> AddrspacePtr { + unsafe { addrspace_ptr_cast(self) } + } + + unsafe fn cast_addrspace( + self, + ) -> AddrspacePtr { + unsafe { addrspace_ptr_cast(self) } + } + + unsafe fn offset(self, count: isize) -> Self { + unsafe { addrspace_ptr_offset(self, count) } + } + + unsafe fn wrapping_offset(self, count: isize) -> Self { + unsafe { addrspace_ptr_arith_offset(self, count) } + } + + unsafe fn read(self) -> T { + unsafe { addrspace_ptr_read_via_copy(self) } + } + + unsafe fn write(self, value: T) { + unsafe { addrspace_ptr_write_via_move(self, value) } + } +} + +impl AddrspacePtr { + unsafe fn from_ptr(ptr: *mut T) -> Self { + unsafe { addrspace_ptr_from_ptr(ptr) } + } + + unsafe fn as_ptr(self) -> *mut T { + unsafe { addrspace_ptr_to_ptr(self) } + } +} + +#[unsafe(no_mangle)] +fn addr(ptr: AddrspacePtr) -> u32 { + // CHECK-LABEL: @addr + // CHECK: %[[val:[^ ]+]] = ptrtoint ptr addrspace(3) %ptr to i32 + // CHECK: ret i32 %[[val]] + unsafe { ptr.addr() } +} + +#[unsafe(no_mangle)] +fn cast( + ptr: AddrspacePtr, +) -> AddrspacePtr { + // CHECK-LABEL: @cast + // CHECK: ret ptr addrspace(3) %ptr + unsafe { ptr.cast() } +} + +#[unsafe(no_mangle)] +fn cast_addrspace( + ptr: AddrspacePtr, +) -> AddrspacePtr { + // CHECK-LABEL: @cast_addrspace + // CHECK: %[[val:[^ ]+]] = addrspacecast ptr addrspace(3) %ptr to ptr + // CHECK: ret ptr %[[val]] + unsafe { ptr.cast_addrspace() } +} + +#[unsafe(no_mangle)] +fn offset( + ptr: AddrspacePtr, +) -> AddrspacePtr { + // CHECK-LABEL: @offset + // CHECK: %[[val:[^ ]+]] = getelementptr inbounds i8, ptr addrspace(3) %ptr, i32 -20 + // CHECK: ret ptr addrspace(3) %[[val]] + unsafe { ptr.offset(-20) } +} + +#[unsafe(no_mangle)] +fn wrapping_offset( + ptr: AddrspacePtr, +) -> AddrspacePtr { + // CHECK-LABEL: @wrapping_offset + // CHECK: %[[val:[^ ]+]] = getelementptr i8, ptr addrspace(3) %ptr, i32 -15 + // CHECK: ret ptr addrspace(3) %[[val]] + unsafe { ptr.wrapping_offset(-15) } +} + +#[unsafe(no_mangle)] +fn read(ptr: AddrspacePtr) -> u32 { + // CHECK-LABEL: @read + // CHECK: %[[val:[^ ]+]] = load i32, ptr addrspace(3) %ptr, align 4 + // CHECK: ret i32 %[[val]] + unsafe { ptr.read() } +} + +#[unsafe(no_mangle)] +fn write(ptr: AddrspacePtr, val: u32) { + // CHECK-LABEL: @write + // CHECK: store i32 %val, ptr addrspace(3) %ptr, align 4 + // CHECK: ret void + unsafe { ptr.write(val) } +} + +#[unsafe(no_mangle)] +fn fun(ptr: *mut u8) -> (AddrspacePtr, usize) { + // CHECK-LABEL: @fun + // CHECK: %[[v0:[^ ]+]] = addrspacecast ptr %ptr to ptr addrspace(3) + // CHECK: %[[v1:[^ ]+]] = insertvalue { ptr addrspace(3), i64 } poison, ptr addrspace(3) %[[v0]], 0 + // CHECK: %[[res:[^ ]+]] = insertvalue { ptr addrspace(3), i64 } %[[v1]], i64 4, 1 + // CHECK: ret { ptr addrspace(3), i64 } %[[res]] + + unsafe { + let p: AddrspacePtr = + AddrspacePtr::from_ptr(ptr).cast_addrspace(); + let size = size_of_val(&p); + (p, size) + } +} + +#[unsafe(no_mangle)] +fn get_raw_dispatch_ptr() -> AddrspacePtr { + // CHECK-LABEL: @get_raw_dispatch_ptr + // CHECK: %[[val:[^ ]+]] = tail call noundef ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() + // CHECK: ret ptr addrspace(4) %[[val]] + unsafe { dispatch_ptr() } +} + +#[unsafe(no_mangle)] +fn get_dispatch_ptr() -> &'static DispatchPacket { + // CHECK-LABEL: @get_dispatch_ptr + // CHECK: %[[val:[^ ]+]] = tail call noundef ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() + // CHECK: %[[res:[^ ]+]] = addrspacecast ptr addrspace(4) %[[val]] to ptr + // CHECK: ret ptr %[[res]] + unsafe { &*dispatch_ptr().cast_addrspace::<{ addrspace::GENERIC }>().as_ptr() } +} + +#[unsafe(no_mangle)] +fn read_ptr( + ptr: AddrspacePtr, +) -> AddrspacePtr { + // Read a pointer to a specific addrspace from a pointer + // CHECK-LABEL: @read + // CHECK: %[[val:[^ ]+]] = load ptr addrspace(3), ptr addrspace(4) %ptr, align 4 + // CHECK: ret ptr addrspace(3) %[[val]] + unsafe { ptr.cast::>().read() } +} + +const EXPECTED: usize = 4; +const ACTUAL: usize = mem::size_of::>(); +// Validate that the size is 4 byte, which is different from the generic pointer size +const _: [(); EXPECTED] = [(); ACTUAL];