rustc_sanitizers/cfi/typeid/itanium_cxx_abi/
transform.rs

1//! Transforms instances and types for LLVM CFI and cross-language LLVM CFI support using Itanium
2//! C++ ABI mangling.
3//!
4//! For more information about LLVM CFI and cross-language LLVM CFI support for the Rust compiler,
5//! see design document in the tracking issue #89653.
6
7use std::iter;
8
9use rustc_hir as hir;
10use rustc_hir::LangItem;
11use rustc_middle::bug;
12use rustc_middle::ty::{
13    self, ExistentialPredicateStableCmpExt as _, Instance, InstanceKind, IntTy, List, TraitRef, Ty,
14    TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt, UintTy,
15};
16use rustc_span::def_id::DefId;
17use rustc_span::sym;
18use rustc_trait_selection::traits;
19use tracing::{debug, instrument};
20
21use crate::cfi::typeid::TypeIdOptions;
22use crate::cfi::typeid::itanium_cxx_abi::encode::EncodeTyOptions;
23
24/// Options for transform_ty.
25pub(crate) type TransformTyOptions = TypeIdOptions;
26
27pub(crate) struct TransformTy<'tcx> {
28    tcx: TyCtxt<'tcx>,
29    options: TransformTyOptions,
30    parents: Vec<Ty<'tcx>>,
31}
32
33impl<'tcx> TransformTy<'tcx> {
34    pub(crate) fn new(tcx: TyCtxt<'tcx>, options: TransformTyOptions) -> Self {
35        TransformTy { tcx, options, parents: Vec::new() }
36    }
37}
38
39/// Transforms a ty:Ty for being encoded and used in the substitution dictionary.
40///
41/// * Transforms all c_void types into unit types.
42/// * Generalizes pointers if TransformTyOptions::GENERALIZE_POINTERS option is set.
43/// * Normalizes integers if TransformTyOptions::NORMALIZE_INTEGERS option is set.
44/// * Generalizes any repr(transparent) user-defined type that is either a pointer or reference, and
45///   either references itself or any other type that contains or references itself, to avoid a
46///   reference cycle.
47/// * Transforms repr(transparent) types without non-ZST field into ().
48///
49impl<'tcx> TypeFolder<TyCtxt<'tcx>> for TransformTy<'tcx> {
50    // Transforms a ty:Ty for being encoded and used in the substitution dictionary.
51    fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
52        match t.kind() {
53            ty::Closure(..)
54            | ty::Coroutine(..)
55            | ty::CoroutineClosure(..)
56            | ty::CoroutineWitness(..)
57            | ty::Dynamic(..)
58            | ty::Float(..)
59            | ty::FnDef(..)
60            | ty::Foreign(..)
61            | ty::Never
62            | ty::Pat(..)
63            | ty::Slice(..)
64            | ty::Str
65            | ty::Tuple(..)
66            | ty::UnsafeBinder(_) => t.super_fold_with(self),
67
68            // Don't transform the type of the array length and keep it as `usize`.
69            // This is required for `try_to_target_usize` to work correctly.
70            &ty::Array(inner, len) => {
71                let inner = self.fold_ty(inner);
72                Ty::new_array_with_const_len(self.tcx, inner, len)
73            }
74
75            ty::Bool => {
76                if self.options.contains(EncodeTyOptions::NORMALIZE_INTEGERS) {
77                    // Note: on all platforms that Rust's currently supports, its size and alignment
78                    // are 1, and its ABI class is INTEGER - see Rust Layout and ABIs.
79                    //
80                    // (See https://rust-lang.github.io/unsafe-code-guidelines/layout/scalars.html#bool.)
81                    //
82                    // Clang represents bool as an 8-bit unsigned integer.
83                    self.tcx.types.u8
84                } else {
85                    t
86                }
87            }
88
89            ty::Char => {
90                if self.options.contains(EncodeTyOptions::NORMALIZE_INTEGERS) {
91                    // Since #118032, char is guaranteed to have the same size, alignment, and
92                    // function call ABI as u32 on all platforms.
93                    self.tcx.types.u32
94                } else {
95                    t
96                }
97            }
98
99            ty::Int(..) | ty::Uint(..) => {
100                if self.options.contains(EncodeTyOptions::NORMALIZE_INTEGERS) {
101                    // Note: C99 7.18.2.4 requires uintptr_t and intptr_t to be at least 16-bit
102                    // wide. All platforms we currently support have a C platform, and as a
103                    // consequence, isize/usize are at least 16-bit wide for all of them.
104                    //
105                    // (See https://rust-lang.github.io/unsafe-code-guidelines/layout/scalars.html#isize-and-usize.)
106                    match t.kind() {
107                        ty::Int(IntTy::Isize) => match self.tcx.sess.target.pointer_width {
108                            16 => self.tcx.types.i16,
109                            32 => self.tcx.types.i32,
110                            64 => self.tcx.types.i64,
111                            128 => self.tcx.types.i128,
112                            _ => bug!(
113                                "fold_ty: unexpected pointer width `{}`",
114                                self.tcx.sess.target.pointer_width
115                            ),
116                        },
117                        ty::Uint(UintTy::Usize) => match self.tcx.sess.target.pointer_width {
118                            16 => self.tcx.types.u16,
119                            32 => self.tcx.types.u32,
120                            64 => self.tcx.types.u64,
121                            128 => self.tcx.types.u128,
122                            _ => bug!(
123                                "fold_ty: unexpected pointer width `{}`",
124                                self.tcx.sess.target.pointer_width
125                            ),
126                        },
127                        _ => t,
128                    }
129                } else {
130                    t
131                }
132            }
133
134            ty::Adt(..) if t.is_c_void(self.tcx) => self.tcx.types.unit,
135
136            ty::Adt(adt_def, args) => {
137                if adt_def.repr().transparent() && adt_def.is_struct() && !self.parents.contains(&t)
138                {
139                    // Don't transform repr(transparent) types with an user-defined CFI encoding to
140                    // preserve the user-defined CFI encoding.
141                    if let Some(_) = self.tcx.get_attr(adt_def.did(), sym::cfi_encoding) {
142                        return t;
143                    }
144                    let variant = adt_def.non_enum_variant();
145                    let typing_env = ty::TypingEnv::post_analysis(self.tcx, variant.def_id);
146                    let field = variant.fields.iter().find(|field| {
147                        let ty = self.tcx.type_of(field.did).instantiate_identity();
148                        let is_zst = self
149                            .tcx
150                            .layout_of(typing_env.as_query_input(ty))
151                            .is_ok_and(|layout| layout.is_zst());
152                        !is_zst
153                    });
154                    if let Some(field) = field {
155                        let ty0 = self.tcx.normalize_erasing_regions(
156                            ty::TypingEnv::fully_monomorphized(),
157                            field.ty(self.tcx, args),
158                        );
159                        // Generalize any repr(transparent) user-defined type that is either a
160                        // pointer or reference, and either references itself or any other type that
161                        // contains or references itself, to avoid a reference cycle.
162
163                        // If the self reference is not through a pointer, for example, due
164                        // to using `PhantomData`, need to skip normalizing it if we hit it again.
165                        self.parents.push(t);
166                        let ty = if ty0.is_any_ptr() && ty0.contains(t) {
167                            let options = self.options;
168                            self.options |= TransformTyOptions::GENERALIZE_POINTERS;
169                            let ty = ty0.fold_with(self);
170                            self.options = options;
171                            ty
172                        } else {
173                            ty0.fold_with(self)
174                        };
175                        self.parents.pop();
176                        ty
177                    } else {
178                        // Transform repr(transparent) types without non-ZST field into ()
179                        self.tcx.types.unit
180                    }
181                } else {
182                    t.super_fold_with(self)
183                }
184            }
185
186            ty::Ref(..) => {
187                if self.options.contains(TransformTyOptions::GENERALIZE_POINTERS) {
188                    if t.is_mutable_ptr() {
189                        Ty::new_mut_ref(self.tcx, self.tcx.lifetimes.re_static, self.tcx.types.unit)
190                    } else {
191                        Ty::new_imm_ref(self.tcx, self.tcx.lifetimes.re_static, self.tcx.types.unit)
192                    }
193                } else {
194                    t.super_fold_with(self)
195                }
196            }
197
198            ty::RawPtr(..) => {
199                if self.options.contains(TransformTyOptions::GENERALIZE_POINTERS) {
200                    if t.is_mutable_ptr() {
201                        Ty::new_mut_ptr(self.tcx, self.tcx.types.unit)
202                    } else {
203                        Ty::new_imm_ptr(self.tcx, self.tcx.types.unit)
204                    }
205                } else {
206                    t.super_fold_with(self)
207                }
208            }
209
210            ty::FnPtr(..) => {
211                if self.options.contains(TransformTyOptions::GENERALIZE_POINTERS) {
212                    Ty::new_imm_ptr(self.tcx, self.tcx.types.unit)
213                } else {
214                    t.super_fold_with(self)
215                }
216            }
217
218            ty::Alias(..) => self.fold_ty(
219                self.tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), t),
220            ),
221
222            ty::Bound(..) | ty::Error(..) | ty::Infer(..) | ty::Param(..) | ty::Placeholder(..) => {
223                bug!("fold_ty: unexpected `{:?}`", t.kind());
224            }
225        }
226    }
227
228    fn cx(&self) -> TyCtxt<'tcx> {
229        self.tcx
230    }
231}
232
233#[instrument(skip(tcx), ret)]
234fn trait_object_ty<'tcx>(tcx: TyCtxt<'tcx>, poly_trait_ref: ty::PolyTraitRef<'tcx>) -> Ty<'tcx> {
235    assert!(!poly_trait_ref.has_non_region_param());
236    let principal_pred = poly_trait_ref.map_bound(|trait_ref| {
237        ty::ExistentialPredicate::Trait(ty::ExistentialTraitRef::erase_self_ty(tcx, trait_ref))
238    });
239    let mut assoc_preds: Vec<_> = traits::supertraits(tcx, poly_trait_ref)
240        .flat_map(|super_poly_trait_ref| {
241            tcx.associated_items(super_poly_trait_ref.def_id())
242                .in_definition_order()
243                .filter(|item| item.kind == ty::AssocKind::Type)
244                .filter(|item| !tcx.generics_require_sized_self(item.def_id))
245                .map(move |assoc_ty| {
246                    super_poly_trait_ref.map_bound(|super_trait_ref| {
247                        let alias_ty =
248                            ty::AliasTy::new_from_args(tcx, assoc_ty.def_id, super_trait_ref.args);
249                        let resolved = tcx.normalize_erasing_regions(
250                            ty::TypingEnv::fully_monomorphized(),
251                            alias_ty.to_ty(tcx),
252                        );
253                        debug!("Resolved {:?} -> {resolved}", alias_ty.to_ty(tcx));
254                        ty::ExistentialPredicate::Projection(
255                            ty::ExistentialProjection::erase_self_ty(
256                                tcx,
257                                ty::ProjectionPredicate {
258                                    projection_term: alias_ty.into(),
259                                    term: resolved.into(),
260                                },
261                            ),
262                        )
263                    })
264                })
265        })
266        .collect();
267    assoc_preds.sort_by(|a, b| a.skip_binder().stable_cmp(tcx, &b.skip_binder()));
268    let preds = tcx.mk_poly_existential_predicates_from_iter(
269        iter::once(principal_pred).chain(assoc_preds.into_iter()),
270    );
271    Ty::new_dynamic(tcx, preds, tcx.lifetimes.re_erased, ty::Dyn)
272}
273
274/// Transforms an instance for LLVM CFI and cross-language LLVM CFI support using Itanium C++ ABI
275/// mangling.
276///
277/// typeid_for_instance is called at two locations, initially when declaring/defining functions and
278/// methods, and later during code generation at call sites, after type erasure might have occurred.
279///
280/// In the first call (i.e., when declaring/defining functions and methods), it encodes type ids for
281/// an FnAbi or Instance, and these type ids are attached to functions and methods. (These type ids
282/// are used later by the LowerTypeTests LLVM pass to aggregate functions in groups derived from
283/// these type ids.)
284///
285/// In the second call (i.e., during code generation at call sites), it encodes a type id for an
286/// FnAbi or Instance, after type erasure might have occurred, and this type id is used for testing
287/// if a function is member of the group derived from this type id. Therefore, in the first call to
288/// typeid_for_fnabi (when type ids are attached to functions and methods), it can only include at
289/// most as much information that would be available in the second call (i.e., during code
290/// generation at call sites); otherwise, the type ids would not match.
291///
292/// For this, it:
293///
294/// * Adjust the type ids of DropGlues (see below).
295/// * Adjusts the type ids of VTableShims to the type id expected in the call sites for the
296///   entry in the vtable (i.e., by using the signature of the closure passed as an argument to the
297///   shim, or by just removing self).
298/// * Performs type erasure for calls on trait objects by transforming self into a trait object of
299///   the trait that defines the method.
300/// * Performs type erasure for closures call methods by transforming self into a trait object of
301///   the Fn trait that defines the method (for being attached as a secondary type id).
302///
303#[instrument(level = "trace", skip(tcx))]
304pub(crate) fn transform_instance<'tcx>(
305    tcx: TyCtxt<'tcx>,
306    mut instance: Instance<'tcx>,
307    options: TransformTyOptions,
308) -> Instance<'tcx> {
309    // FIXME: account for async-drop-glue
310    if (matches!(instance.def, ty::InstanceKind::Virtual(..))
311        && tcx.is_lang_item(instance.def_id(), LangItem::DropInPlace))
312        || matches!(instance.def, ty::InstanceKind::DropGlue(..))
313    {
314        // Adjust the type ids of DropGlues
315        //
316        // DropGlues may have indirect calls to one or more given types drop function. Rust allows
317        // for types to be erased to any trait object and retains the drop function for the original
318        // type, which means at the indirect call sites in DropGlues, when typeid_for_fnabi is
319        // called a second time, it only has information after type erasure and it could be a call
320        // on any arbitrary trait object. Normalize them to a synthesized Drop trait object, both on
321        // declaration/definition, and during code generation at call sites so they have the same
322        // type id and match.
323        //
324        // FIXME(rcvalle): This allows a drop call on any trait object to call the drop function of
325        //   any other type.
326        //
327        let def_id = tcx
328            .lang_items()
329            .drop_trait()
330            .unwrap_or_else(|| bug!("typeid_for_instance: couldn't get drop_trait lang item"));
331        let predicate = ty::ExistentialPredicate::Trait(ty::ExistentialTraitRef::new_from_args(
332            tcx,
333            def_id,
334            ty::List::empty(),
335        ));
336        let predicates = tcx.mk_poly_existential_predicates(&[ty::Binder::dummy(predicate)]);
337        let self_ty = Ty::new_dynamic(tcx, predicates, tcx.lifetimes.re_erased, ty::Dyn);
338        instance.args = tcx.mk_args_trait(self_ty, List::empty());
339    } else if let ty::InstanceKind::Virtual(def_id, _) = instance.def {
340        // Transform self into a trait object of the trait that defines the method for virtual
341        // functions to match the type erasure done below.
342        let upcast_ty = match tcx.trait_of_item(def_id) {
343            Some(trait_id) => trait_object_ty(
344                tcx,
345                ty::Binder::dummy(ty::TraitRef::from_method(tcx, trait_id, instance.args)),
346            ),
347            // drop_in_place won't have a defining trait, skip the upcast
348            None => instance.args.type_at(0),
349        };
350        let ty::Dynamic(preds, lifetime, kind) = upcast_ty.kind() else {
351            bug!("Tried to remove autotraits from non-dynamic type {upcast_ty}");
352        };
353        let self_ty = if preds.principal().is_some() {
354            let filtered_preds =
355                tcx.mk_poly_existential_predicates_from_iter(preds.into_iter().filter(|pred| {
356                    !matches!(pred.skip_binder(), ty::ExistentialPredicate::AutoTrait(..))
357                }));
358            Ty::new_dynamic(tcx, filtered_preds, *lifetime, *kind)
359        } else {
360            // If there's no principal type, re-encode it as a unit, since we don't know anything
361            // about it. This technically discards the knowledge that it was a type that was made
362            // into a trait object at some point, but that's not a lot.
363            tcx.types.unit
364        };
365        instance.args = tcx.mk_args_trait(self_ty, instance.args.into_iter().skip(1));
366    } else if let ty::InstanceKind::VTableShim(def_id) = instance.def
367        && let Some(trait_id) = tcx.trait_of_item(def_id)
368    {
369        // Adjust the type ids of VTableShims to the type id expected in the call sites for the
370        // entry in the vtable (i.e., by using the signature of the closure passed as an argument
371        // to the shim, or by just removing self).
372        let trait_ref = ty::TraitRef::new_from_args(tcx, trait_id, instance.args);
373        let invoke_ty = trait_object_ty(tcx, ty::Binder::dummy(trait_ref));
374        instance.args = tcx.mk_args_trait(invoke_ty, trait_ref.args.into_iter().skip(1));
375    }
376
377    if !options.contains(TransformTyOptions::USE_CONCRETE_SELF) {
378        // Perform type erasure for calls on trait objects by transforming self into a trait object
379        // of the trait that defines the method.
380        if let Some((trait_ref, method_id, ancestor)) = implemented_method(tcx, instance) {
381            // Trait methods will have a Self polymorphic parameter, where the concreteized
382            // implementation will not. We need to walk back to the more general trait method
383            let trait_ref = tcx.instantiate_and_normalize_erasing_regions(
384                instance.args,
385                ty::TypingEnv::fully_monomorphized(),
386                trait_ref,
387            );
388            let invoke_ty = trait_object_ty(tcx, ty::Binder::dummy(trait_ref));
389
390            // At the call site, any call to this concrete function through a vtable will be
391            // `Virtual(method_id, idx)` with appropriate arguments for the method. Since we have the
392            // original method id, and we've recovered the trait arguments, we can make the callee
393            // instance we're computing the alias set for match the caller instance.
394            //
395            // Right now, our code ignores the vtable index everywhere, so we use 0 as a placeholder.
396            // If we ever *do* start encoding the vtable index, we will need to generate an alias set
397            // based on which vtables we are putting this method into, as there will be more than one
398            // index value when supertraits are involved.
399            instance.def = ty::InstanceKind::Virtual(method_id, 0);
400            let abstract_trait_args =
401                tcx.mk_args_trait(invoke_ty, trait_ref.args.into_iter().skip(1));
402            instance.args = instance.args.rebase_onto(tcx, ancestor, abstract_trait_args);
403        } else if tcx.is_closure_like(instance.def_id()) {
404            // We're either a closure or a coroutine. Our goal is to find the trait we're defined on,
405            // instantiate it, and take the type of its only method as our own.
406            let closure_ty = instance.ty(tcx, ty::TypingEnv::fully_monomorphized());
407            let (trait_id, inputs) = match closure_ty.kind() {
408                ty::Closure(..) => {
409                    let closure_args = instance.args.as_closure();
410                    let trait_id = tcx.fn_trait_kind_to_def_id(closure_args.kind()).unwrap();
411                    let tuple_args =
412                        tcx.instantiate_bound_regions_with_erased(closure_args.sig()).inputs()[0];
413                    (trait_id, Some(tuple_args))
414                }
415                ty::Coroutine(..) => match tcx.coroutine_kind(instance.def_id()).unwrap() {
416                    hir::CoroutineKind::Coroutine(..) => (
417                        tcx.require_lang_item(LangItem::Coroutine, None),
418                        Some(instance.args.as_coroutine().resume_ty()),
419                    ),
420                    hir::CoroutineKind::Desugared(desugaring, _) => {
421                        let lang_item = match desugaring {
422                            hir::CoroutineDesugaring::Async => LangItem::Future,
423                            hir::CoroutineDesugaring::AsyncGen => LangItem::AsyncIterator,
424                            hir::CoroutineDesugaring::Gen => LangItem::Iterator,
425                        };
426                        (tcx.require_lang_item(lang_item, None), None)
427                    }
428                },
429                ty::CoroutineClosure(..) => (
430                    tcx.require_lang_item(LangItem::FnOnce, None),
431                    Some(
432                        tcx.instantiate_bound_regions_with_erased(
433                            instance.args.as_coroutine_closure().coroutine_closure_sig(),
434                        )
435                        .tupled_inputs_ty,
436                    ),
437                ),
438                x => bug!("Unexpected type kind for closure-like: {x:?}"),
439            };
440            let concrete_args = tcx.mk_args_trait(closure_ty, inputs.map(Into::into));
441            let trait_ref = ty::TraitRef::new_from_args(tcx, trait_id, concrete_args);
442            let invoke_ty = trait_object_ty(tcx, ty::Binder::dummy(trait_ref));
443            let abstract_args = tcx.mk_args_trait(invoke_ty, trait_ref.args.into_iter().skip(1));
444            // There should be exactly one method on this trait, and it should be the one we're
445            // defining.
446            let call = tcx
447                .associated_items(trait_id)
448                .in_definition_order()
449                .find(|it| it.kind == ty::AssocKind::Fn)
450                .expect("No call-family function on closure-like Fn trait?")
451                .def_id;
452
453            instance.def = ty::InstanceKind::Virtual(call, 0);
454            instance.args = abstract_args;
455        }
456    }
457
458    instance
459}
460
461fn implemented_method<'tcx>(
462    tcx: TyCtxt<'tcx>,
463    instance: Instance<'tcx>,
464) -> Option<(ty::EarlyBinder<'tcx, TraitRef<'tcx>>, DefId, DefId)> {
465    let trait_ref;
466    let method_id;
467    let trait_id;
468    let trait_method;
469    let ancestor = if let Some(impl_id) = tcx.impl_of_method(instance.def_id()) {
470        // Implementation in an `impl` block
471        trait_ref = tcx.impl_trait_ref(impl_id)?;
472        let impl_method = tcx.associated_item(instance.def_id());
473        method_id = impl_method.trait_item_def_id?;
474        trait_method = tcx.associated_item(method_id);
475        trait_id = trait_ref.skip_binder().def_id;
476        impl_id
477    } else if let InstanceKind::Item(def_id) = instance.def
478        && let Some(trait_method_bound) = tcx.opt_associated_item(def_id)
479    {
480        // Provided method in a `trait` block
481        trait_method = trait_method_bound;
482        method_id = instance.def_id();
483        trait_id = tcx.trait_of_item(method_id)?;
484        trait_ref = ty::EarlyBinder::bind(TraitRef::from_method(tcx, trait_id, instance.args));
485        trait_id
486    } else {
487        return None;
488    };
489    let vtable_possible = traits::is_vtable_safe_method(tcx, trait_id, trait_method)
490        && tcx.is_dyn_compatible(trait_id);
491    vtable_possible.then_some((trait_ref, method_id, ancestor))
492}