Add AddrspacePtr for pointers to non-0 addrspaces

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -197,6 +197,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     &[ptr, args[1].immediate()],
                 )
             }
+            sym::addrspace_ptr_cast | sym::addrspace_ptr_from_ptr | sym::addrspace_ptr_to_ptr => {
+                self.pointercast(args[0].immediate(), result.layout.immediate_llvm_type(self.cx))
+            }
+            sym::addrspace_ptr_to_addr => {
+                self.ptrtoint(args[0].immediate(), result.layout.immediate_llvm_type(self.cx))
+            }
             sym::autodiff => {
                 codegen_autodiff(self, tcx, instance, args, result);
                 return Ok(());

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -186,7 +186,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
                 value
             }
-            sym::arith_offset => {
+            sym::arith_offset | sym::addrspace_ptr_arith_offset => {
                 let ty = fn_args.type_at(0);
                 let layout = bx.layout_of(ty);
                 let ptr = args[0].immediate();

compiler/rustc_hir/src/lang_items.rs

@@ -186,6 +186,7 @@ language_item_table! {
     PointeeTrait,            sym::pointee_trait,       pointee_trait,              Target::Trait,          GenericRequirement::None;
     Metadata,                sym::metadata_type,       metadata_type,              Target::AssocTy,        GenericRequirement::None;
     DynMetadata,             sym::dyn_metadata,        dyn_metadata,               Target::Struct,         GenericRequirement::None;
+    AddrspacePtr,            sym::addrspace_ptr_type,  addrspace_ptr_type,         Target::Struct,         GenericRequirement::Exact(2);
 
     Freeze,                  sym::freeze,              freeze_trait,               Target::Trait,          GenericRequirement::Exact(0);
     UnsafeUnpin,             sym::unsafe_unpin,        unsafe_unpin_trait,         Target::Trait,          GenericRequirement::Exact(0);

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -4,7 +4,7 @@ use rustc_abi::ExternAbi;
 use rustc_errors::DiagMessage;
 use rustc_hir::{self as hir, LangItem};
 use rustc_middle::traits::{ObligationCause, ObligationCauseCode};
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{self, Const, Ty, TyCtxt};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{Span, Symbol, sym};
 
@@ -239,8 +239,8 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi
 
 /// Remember to add all intrinsics here, in `compiler/rustc_codegen_llvm/src/intrinsic.rs`,
 /// and in `library/core/src/intrinsics.rs`.
-pub(crate) fn check_intrinsic_type(
-    tcx: TyCtxt<'_>,
+pub(crate) fn check_intrinsic_type<'tcx>(
+    tcx: TyCtxt<'tcx>,
     intrinsic_id: LocalDefId,
     span: Span,
     intrinsic_name: Symbol,
@@ -255,6 +255,15 @@ pub(crate) fn check_intrinsic_type(
             Ty::new_error_with_message(tcx, span, "expected param")
         }
     };
+    let const_param = |n| {
+        if let &ty::GenericParamDef { name, kind: ty::GenericParamDefKind::Const { .. }, .. } =
+            generics.param_at(n as usize, tcx)
+        {
+            Const::new_param(tcx, ty::ParamConst::new(n, name))
+        } else {
+            Const::new_error_with_message(tcx, span, "expected const param")
+        }
+    };
 
     let bound_vars = tcx.mk_bound_variable_kinds(&[
         ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon),
@@ -280,9 +289,49 @@ pub(crate) fn check_intrinsic_type(
         (Ty::new_ref(tcx, env_region, va_list_ty, mutbl), va_list_ty)
     };
 
+    let mk_u32_const =
+        |i: u32| Const::new_value(tcx, ty::ValTree::from_scalar_int(tcx, i.into()), tcx.types.u32);
+    let mk_addrspace_ptr = |t: Ty<'tcx>, addrspace: Const<'tcx>| {
+        tcx.type_of(tcx.lang_items().addrspace_ptr_type().unwrap())
+            .instantiate(tcx, &[t.into(), addrspace.into()])
+    };
+
     let safety = intrinsic_operation_unsafety(tcx, intrinsic_id);
     let n_lts = 0;
     let (n_tps, n_cts, inputs, output) = match intrinsic_name {
+        sym::addrspace_ptr_arith_offset | sym::addrspace_ptr_offset => (
+            1,
+            1,
+            vec![mk_addrspace_ptr(param(0), const_param(1)), tcx.types.isize],
+            mk_addrspace_ptr(param(0), const_param(1)),
+        ),
+        sym::addrspace_ptr_cast => (
+            2,
+            2,
+            vec![mk_addrspace_ptr(param(0), const_param(2))],
+            mk_addrspace_ptr(param(1), const_param(3)),
+        ),
+        sym::addrspace_ptr_to_addr => {
+            (1, 1, vec![mk_addrspace_ptr(tcx.types.unit, const_param(1))], param(0))
+        }
+        sym::addrspace_ptr_from_ptr => (
+            1,
+            0,
+            vec![Ty::new_mut_ptr(tcx, param(0).into())],
+            mk_addrspace_ptr(param(0), mk_u32_const(0)),
+        ),
+        sym::addrspace_ptr_to_ptr => (
+            1,
+            0,
+            vec![mk_addrspace_ptr(param(0), mk_u32_const(0))],
+            Ty::new_mut_ptr(tcx, param(0)),
+        ),
+        sym::addrspace_ptr_read_via_copy => {
+            (1, 1, vec![mk_addrspace_ptr(param(0), const_param(1))], param(0))
+        }
+        sym::addrspace_ptr_write_via_move => {
+            (1, 1, vec![mk_addrspace_ptr(param(0), const_param(1)), param(0)], tcx.types.unit)
+        }
         sym::autodiff => (4, 0, vec![param(0), param(1), param(2)], param(3)),
         sym::abort => (0, 0, vec![], tcx.types.never),
         sym::unreachable => (0, 0, vec![], tcx.types.never),

compiler/rustc_hir_typeck/src/place_op.rs

@@ -23,7 +23,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         oprnd_expr: &'tcx hir::Expr<'tcx>,
         oprnd_ty: Ty<'tcx>,
     ) -> Option<Ty<'tcx>> {
-        if let Some(ty) = oprnd_ty.builtin_deref(true) {
+        // AddrspacePtr deref is used internally but it should not be accessed from the Rust,
+        // so exclude it here.
+        if let Some(ty) = oprnd_ty.builtin_deref(true)
+            && !oprnd_ty.is_addrspace_ptr()
+        {
             return Some(ty);
         }
 

compiler/rustc_lint/src/types/improper_ctypes.rs

@@ -468,6 +468,9 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
                 if def.is_phantom_data() {
                     return FfiPhantom(ty);
                 }
+                if def.is_addrspace_ptr() {
+                    return FfiSafe;
+                }
                 match def.adt_kind() {
                     AdtKind::Struct | AdtKind::Union => {
                         if let Some(sym::cstring_type | sym::cstr_type) =

compiler/rustc_middle/src/ty/adt.rs

@@ -59,6 +59,8 @@ bitflags::bitflags! {
         const IS_PIN                        = 1 << 11;
         /// Indicates whether the type is `#[pin_project]`.
         const IS_PIN_PROJECT                = 1 << 12;
+        /// Indicates whether the type is `AddrspacePtr`.
+        const IS_ADDRSPACE_PTR              = 1 << 13;
     }
 }
 rustc_data_structures::external_bitflags_debug! { AdtFlags }
@@ -324,6 +326,9 @@ impl AdtDefData {
         if tcx.is_lang_item(did, LangItem::Pin) {
             flags |= AdtFlags::IS_PIN;
         }
+        if tcx.is_lang_item(did, LangItem::AddrspacePtr) {
+            flags |= AdtFlags::IS_ADDRSPACE_PTR;
+        }
 
         AdtDefData { did, variants, flags, repr }
     }
@@ -445,6 +450,12 @@ impl<'tcx> AdtDef<'tcx> {
         self.flags().contains(AdtFlags::IS_PIN)
     }
 
+    /// Returns `true` if this is `AddrspacePtr<T, ADDRSPACE>`.
+    #[inline]
+    pub fn is_addrspace_ptr(self) -> bool {
+        self.flags().contains(AdtFlags::IS_ADDRSPACE_PTR)
+    }
+
     /// Returns `true` is this is `#[pin_v2]` for the purposes
     /// of structural pinning.
     #[inline]

compiler/rustc_middle/src/ty/sty.rs

@@ -1339,6 +1339,14 @@ impl<'tcx> Ty<'tcx> {
         }
     }
 
+    #[inline]
+    pub fn is_addrspace_ptr(self) -> bool {
+        match self.kind() {
+            Adt(def, _) => def.is_addrspace_ptr(),
+            _ => false,
+        }
+    }
+
     /// Tests whether this is a Box definitely using the global allocator.
     ///
     /// If the allocator is still generic, the answer is `false`, but it may
@@ -1366,6 +1374,13 @@ impl<'tcx> Ty<'tcx> {
         }
     }
 
+    pub fn addrspace_ptr_ty(self) -> Option<Ty<'tcx>> {
+        match self.kind() {
+            Adt(def, args) if def.is_addrspace_ptr() => Some(args.type_at(0)),
+            _ => None,
+        }
+    }
+
     pub fn pinned_ty(self) -> Option<Ty<'tcx>> {
         match self.kind() {
             Adt(def, args) if def.is_pin() => Some(args.type_at(0)),
@@ -1566,6 +1581,11 @@ impl<'tcx> Ty<'tcx> {
     pub fn builtin_deref(self, explicit: bool) -> Option<Ty<'tcx>> {
         match *self.kind() {
             _ if let Some(boxed) = self.boxed_ty() => Some(boxed),
+            _ if let Some(ty) = self.addrspace_ptr_ty()
+                && explicit =>
+            {
+                Some(ty)
+            }
             Ref(_, ty, _) => Some(ty),
             RawPtr(ty, _) if explicit => Some(ty),
             _ => None,

compiler/rustc_mir_transform/src/lower_intrinsics.rs

@@ -145,7 +145,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                         ));
                         terminator.kind = TerminatorKind::Goto { target };
                     }
-                    sym::read_via_copy => {
+                    sym::read_via_copy | sym::addrspace_ptr_read_via_copy => {
                         let Ok([arg]) = take_array(args) else {
                             span_bug!(terminator.source_info.span, "Wrong number of arguments");
                         };
@@ -177,7 +177,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                             Some(target) => TerminatorKind::Goto { target },
                         }
                     }
-                    sym::write_via_move => {
+                    sym::write_via_move | sym::addrspace_ptr_write_via_move => {
                         let target = target.unwrap();
                         let Ok([ptr, val]) = take_array(args) else {
                             span_bug!(
@@ -220,7 +220,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                         ));
                         terminator.kind = TerminatorKind::Goto { target };
                     }
-                    sym::offset => {
+                    sym::offset | sym::addrspace_ptr_offset => {
                         let target = target.unwrap();
                         let Ok([ptr, delta]) = take_array(args) else {
                             span_bug!(

compiler/rustc_mir_transform/src/shim.rs

@@ -543,6 +543,7 @@ fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -
 
     match self_ty.kind() {
         ty::FnDef(..) | ty::FnPtr(..) => builder.copy_shim(),
+        ty::Adt(def, _) if def.is_addrspace_ptr() => builder.copy_shim(),
         ty::Closure(_, args) => builder.tuple_like_shim(dest, src, args.as_closure().upvar_tys()),
         ty::CoroutineClosure(_, args) => {
             builder.tuple_like_shim(dest, src, args.as_coroutine_closure().upvar_tys())

compiler/rustc_mir_transform/src/validate.rs

@@ -1178,7 +1178,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
 
                 match op {
                     Offset => {
-                        check_kinds!(a, "Cannot offset non-pointer type {:?}", ty::RawPtr(..));
+                        if !a.is_addrspace_ptr() {
+                            check_kinds!(a, "Cannot offset non-pointer type {:?}", ty::RawPtr(..));
+                        }
                         if b != self.tcx.types.isize && b != self.tcx.types.usize {
                             self.fail(location, format!("Cannot offset by non-isize type {b}"));
                         }

compiler/rustc_span/src/symbol.rs

@@ -428,6 +428,15 @@ symbols! {
         add_assign,
         add_with_overflow,
         address,
+        addrspace_ptr_arith_offset,
+        addrspace_ptr_cast,
+        addrspace_ptr_from_ptr,
+        addrspace_ptr_offset,
+        addrspace_ptr_read_via_copy,
+        addrspace_ptr_to_addr,
+        addrspace_ptr_to_ptr,
+        addrspace_ptr_type,
+        addrspace_ptr_write_via_move,
         adt_const_params,
         advanced_slice_patterns,
         adx_target_feature,

compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs

@@ -1207,6 +1207,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 candidates.vec.push(SizedCandidate);
             }
 
+            ty::Adt(def, _) if def.is_addrspace_ptr() => {
+                candidates.vec.push(BuiltinCandidate);
+            }
+
             // Fallback to whatever user-defined impls or param-env clauses exist in this case.
             ty::Adt(..) | ty::Alias(..) | ty::Param(..) | ty::Placeholder(..) => {}
 

compiler/rustc_trait_selection/src/traits/select/mod.rs

@@ -2256,6 +2256,8 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 ty::Binder::dummy(args.as_coroutine_closure().upvar_tys().to_vec())
             }
 
+            ty::Adt(def, args) if def.is_addrspace_ptr() => ty::Binder::dummy(vec![]),
+
             ty::Foreign(..)
             | ty::Str
             | ty::Slice(_)

compiler/rustc_ty_utils/src/instance.rs

@@ -268,6 +268,7 @@ fn resolve_associated_item<'tcx>(
                         | ty::Closure(..)
                         | ty::CoroutineClosure(..)
                         | ty::Tuple(..) => {}
+                        ty::Adt(def, _) if def.is_addrspace_ptr() => {}
                         _ => return Ok(None),
                     };
 

compiler/rustc_ty_utils/src/layout.rs

@@ -643,6 +643,21 @@ fn layout_of_uncached<'tcx>(
 
         // ADTs.
         ty::Adt(def, args) => {
+            if def.is_addrspace_ptr() {
+                // Implement the AddrspacePtr lang item struct as a pointer into the
+                // address space from the const generic argument.
+                let pointee = args.type_at(0);
+                let addrspace = extract_const_value(cx, ty, args.const_at(1))?
+                    .try_to_bits(tcx, ty::TypingEnv::fully_monomorphized())
+                    .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?
+                    as u32;
+                let data_ptr = scalar_unit(Pointer(AddressSpace(addrspace)));
+                if !pointee.is_sized(tcx, cx.typing_env) {
+                    panic!("AddrspacePtr pointee must be sized");
+                }
+                return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr)));
+            }
+
             // Cache the field layouts.
             let variants = def
                 .variants()