rustc_borrowck/type_check/
mod.rs

1//! This pass type-checks the MIR to ensure it is not broken.
2
3use std::rc::Rc;
4use std::{fmt, iter, mem};
5
6use rustc_abi::FieldIdx;
7use rustc_data_structures::frozen::Frozen;
8use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
9use rustc_errors::ErrorGuaranteed;
10use rustc_hir as hir;
11use rustc_hir::def::DefKind;
12use rustc_hir::def_id::LocalDefId;
13use rustc_hir::lang_items::LangItem;
14use rustc_index::{IndexSlice, IndexVec};
15use rustc_infer::infer::canonical::QueryRegionConstraints;
16use rustc_infer::infer::outlives::env::RegionBoundPairs;
17use rustc_infer::infer::region_constraints::RegionConstraintData;
18use rustc_infer::infer::{
19    BoundRegion, BoundRegionConversionTime, InferCtxt, NllRegionVariableOrigin,
20};
21use rustc_infer::traits::PredicateObligations;
22use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor};
23use rustc_middle::mir::*;
24use rustc_middle::traits::query::NoSolution;
25use rustc_middle::ty::adjustment::PointerCoercion;
26use rustc_middle::ty::cast::CastTy;
27use rustc_middle::ty::{
28    self, Binder, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, CoroutineArgsExt,
29    Dynamic, GenericArgsRef, OpaqueHiddenType, OpaqueTypeKey, RegionVid, Ty, TyCtxt,
30    TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions,
31};
32use rustc_middle::{bug, span_bug};
33use rustc_mir_dataflow::ResultsCursor;
34use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
35use rustc_mir_dataflow::move_paths::MoveData;
36use rustc_mir_dataflow::points::DenseLocationMap;
37use rustc_span::def_id::CRATE_DEF_ID;
38use rustc_span::source_map::Spanned;
39use rustc_span::{Span, sym};
40use rustc_trait_selection::traits::query::type_op::custom::scrape_region_constraints;
41use rustc_trait_selection::traits::query::type_op::{TypeOp, TypeOpOutput};
42use tracing::{debug, instrument, trace};
43
44use crate::borrow_set::BorrowSet;
45use crate::constraints::{OutlivesConstraint, OutlivesConstraintSet};
46use crate::diagnostics::UniverseInfo;
47use crate::member_constraints::MemberConstraintSet;
48use crate::opaque_types::ConcreteOpaqueTypes;
49use crate::polonius::legacy::{PoloniusFacts, PoloniusLocationTable};
50use crate::polonius::{PoloniusContext, PoloniusLivenessContext};
51use crate::region_infer::TypeTest;
52use crate::region_infer::values::{LivenessValues, PlaceholderIndex, PlaceholderIndices};
53use crate::session_diagnostics::{MoveUnsized, SimdIntrinsicArgConst};
54use crate::type_check::free_region_relations::{CreateResult, UniversalRegionRelations};
55use crate::universal_regions::{DefiningTy, UniversalRegions};
56use crate::{BorrowckInferCtxt, path_utils};
57
58macro_rules! span_mirbug {
59    ($context:expr, $elem:expr, $($message:tt)*) => ({
60        $crate::type_check::mirbug(
61            $context.tcx(),
62            $context.last_span,
63            format!(
64                "broken MIR in {:?} ({:?}): {}",
65                $context.body().source.def_id(),
66                $elem,
67                format_args!($($message)*),
68            ),
69        )
70    })
71}
72
73mod canonical;
74mod constraint_conversion;
75pub(crate) mod free_region_relations;
76mod input_output;
77pub(crate) mod liveness;
78mod opaque_types;
79mod relate_tys;
80
81/// Type checks the given `mir` in the context of the inference
82/// context `infcx`. Returns any region constraints that have yet to
83/// be proven. This result includes liveness constraints that
84/// ensure that regions appearing in the types of all local variables
85/// are live at all points where that local variable may later be
86/// used.
87///
88/// This phase of type-check ought to be infallible -- this is because
89/// the original, HIR-based type-check succeeded. So if any errors
90/// occur here, we will get a `bug!` reported.
91///
92/// # Parameters
93///
94/// - `infcx` -- inference context to use
95/// - `body` -- MIR body to type-check
96/// - `promoted` -- map of promoted constants within `body`
97/// - `universal_regions` -- the universal regions from `body`s function signature
98/// - `location_table` -- for datalog polonius, the map between `Location`s and `RichLocation`s
99/// - `borrow_set` -- information about borrows occurring in `body`
100/// - `polonius_facts` -- when using Polonius, this is the generated set of Polonius facts
101/// - `flow_inits` -- results of a maybe-init dataflow analysis
102/// - `move_data` -- move-data constructed when performing the maybe-init dataflow analysis
103/// - `location_map` -- map between MIR `Location` and `PointIndex`
104pub(crate) fn type_check<'a, 'tcx>(
105    infcx: &BorrowckInferCtxt<'tcx>,
106    body: &Body<'tcx>,
107    promoted: &IndexSlice<Promoted, Body<'tcx>>,
108    universal_regions: UniversalRegions<'tcx>,
109    location_table: &PoloniusLocationTable,
110    borrow_set: &BorrowSet<'tcx>,
111    polonius_facts: &mut Option<PoloniusFacts>,
112    flow_inits: ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
113    move_data: &MoveData<'tcx>,
114    location_map: Rc<DenseLocationMap>,
115    concrete_opaque_types: &mut ConcreteOpaqueTypes<'tcx>,
116) -> MirTypeckResults<'tcx> {
117    let implicit_region_bound = ty::Region::new_var(infcx.tcx, universal_regions.fr_fn_body);
118    let mut constraints = MirTypeckRegionConstraints {
119        placeholder_indices: PlaceholderIndices::default(),
120        placeholder_index_to_region: IndexVec::default(),
121        liveness_constraints: LivenessValues::with_specific_points(Rc::clone(&location_map)),
122        outlives_constraints: OutlivesConstraintSet::default(),
123        member_constraints: MemberConstraintSet::default(),
124        type_tests: Vec::default(),
125        universe_causes: FxIndexMap::default(),
126    };
127
128    let CreateResult {
129        universal_region_relations,
130        region_bound_pairs,
131        normalized_inputs_and_output,
132        known_type_outlives_obligations,
133    } = free_region_relations::create(
134        infcx,
135        infcx.param_env,
136        implicit_region_bound,
137        universal_regions,
138        &mut constraints,
139    );
140
141    let pre_obligations = infcx.take_registered_region_obligations();
142    assert!(
143        pre_obligations.is_empty(),
144        "there should be no incoming region obligations = {pre_obligations:#?}",
145    );
146
147    debug!(?normalized_inputs_and_output);
148
149    let polonius_liveness = if infcx.tcx.sess.opts.unstable_opts.polonius.is_next_enabled() {
150        Some(PoloniusLivenessContext::default())
151    } else {
152        None
153    };
154
155    let mut typeck = TypeChecker {
156        infcx,
157        last_span: body.span,
158        body,
159        promoted,
160        user_type_annotations: &body.user_type_annotations,
161        region_bound_pairs,
162        known_type_outlives_obligations,
163        implicit_region_bound,
164        reported_errors: Default::default(),
165        universal_regions: &universal_region_relations.universal_regions,
166        location_table,
167        polonius_facts,
168        borrow_set,
169        constraints: &mut constraints,
170        concrete_opaque_types,
171        polonius_liveness,
172    };
173
174    typeck.check_user_type_annotations();
175    typeck.visit_body(body);
176    typeck.equate_inputs_and_outputs(&normalized_inputs_and_output);
177    typeck.check_signature_annotation();
178
179    liveness::generate(&mut typeck, &location_map, flow_inits, move_data);
180
181    let opaque_type_values =
182        opaque_types::take_opaques_and_register_member_constraints(&mut typeck);
183
184    // We're done with typeck, we can finalize the polonius liveness context for region inference.
185    let polonius_context = typeck.polonius_liveness.take().map(|liveness_context| {
186        PoloniusContext::create_from_liveness(
187            liveness_context,
188            infcx.num_region_vars(),
189            typeck.constraints.liveness_constraints.points(),
190        )
191    });
192
193    MirTypeckResults {
194        constraints,
195        universal_region_relations,
196        opaque_type_values,
197        polonius_context,
198    }
199}
200
201#[track_caller]
202fn mirbug(tcx: TyCtxt<'_>, span: Span, msg: String) {
203    // We sometimes see MIR failures (notably predicate failures) due to
204    // the fact that we check rvalue sized predicates here. So use `span_delayed_bug`
205    // to avoid reporting bugs in those cases.
206    tcx.dcx().span_delayed_bug(span, msg);
207}
208
209enum FieldAccessError {
210    OutOfRange { field_count: usize },
211}
212
213/// The MIR type checker. Visits the MIR and enforces all the
214/// constraints needed for it to be valid and well-typed. Along the
215/// way, it accrues region constraints -- these can later be used by
216/// NLL region checking.
217struct TypeChecker<'a, 'tcx> {
218    infcx: &'a BorrowckInferCtxt<'tcx>,
219    last_span: Span,
220    body: &'a Body<'tcx>,
221    /// The bodies of all promoteds. As promoteds have a completely separate CFG
222    /// recursing into them may corrupt your data structures if you're not careful.
223    promoted: &'a IndexSlice<Promoted, Body<'tcx>>,
224    /// User type annotations are shared between the main MIR and the MIR of
225    /// all of the promoted items.
226    user_type_annotations: &'a CanonicalUserTypeAnnotations<'tcx>,
227    region_bound_pairs: RegionBoundPairs<'tcx>,
228    known_type_outlives_obligations: Vec<ty::PolyTypeOutlivesPredicate<'tcx>>,
229    implicit_region_bound: ty::Region<'tcx>,
230    reported_errors: FxIndexSet<(Ty<'tcx>, Span)>,
231    universal_regions: &'a UniversalRegions<'tcx>,
232    location_table: &'a PoloniusLocationTable,
233    polonius_facts: &'a mut Option<PoloniusFacts>,
234    borrow_set: &'a BorrowSet<'tcx>,
235    constraints: &'a mut MirTypeckRegionConstraints<'tcx>,
236    concrete_opaque_types: &'a mut ConcreteOpaqueTypes<'tcx>,
237    /// When using `-Zpolonius=next`, the liveness helper data used to create polonius constraints.
238    polonius_liveness: Option<PoloniusLivenessContext>,
239}
240
241/// Holder struct for passing results from MIR typeck to the rest of the non-lexical regions
242/// inference computation.
243pub(crate) struct MirTypeckResults<'tcx> {
244    pub(crate) constraints: MirTypeckRegionConstraints<'tcx>,
245    pub(crate) universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
246    pub(crate) opaque_type_values: FxIndexMap<OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>>,
247    pub(crate) polonius_context: Option<PoloniusContext>,
248}
249
250/// A collection of region constraints that must be satisfied for the
251/// program to be considered well-typed.
252pub(crate) struct MirTypeckRegionConstraints<'tcx> {
253    /// Maps from a `ty::Placeholder` to the corresponding
254    /// `PlaceholderIndex` bit that we will use for it.
255    ///
256    /// To keep everything in sync, do not insert this set
257    /// directly. Instead, use the `placeholder_region` helper.
258    pub(crate) placeholder_indices: PlaceholderIndices,
259
260    /// Each time we add a placeholder to `placeholder_indices`, we
261    /// also create a corresponding "representative" region vid for
262    /// that wraps it. This vector tracks those. This way, when we
263    /// convert the same `ty::RePlaceholder(p)` twice, we can map to
264    /// the same underlying `RegionVid`.
265    pub(crate) placeholder_index_to_region: IndexVec<PlaceholderIndex, ty::Region<'tcx>>,
266
267    /// In general, the type-checker is not responsible for enforcing
268    /// liveness constraints; this job falls to the region inferencer,
269    /// which performs a liveness analysis. However, in some limited
270    /// cases, the MIR type-checker creates temporary regions that do
271    /// not otherwise appear in the MIR -- in particular, the
272    /// late-bound regions that it instantiates at call-sites -- and
273    /// hence it must report on their liveness constraints.
274    pub(crate) liveness_constraints: LivenessValues,
275
276    pub(crate) outlives_constraints: OutlivesConstraintSet<'tcx>,
277
278    pub(crate) member_constraints: MemberConstraintSet<'tcx, RegionVid>,
279
280    pub(crate) universe_causes: FxIndexMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
281
282    pub(crate) type_tests: Vec<TypeTest<'tcx>>,
283}
284
285impl<'tcx> MirTypeckRegionConstraints<'tcx> {
286    /// Creates a `Region` for a given `PlaceholderRegion`, or returns the
287    /// region that corresponds to a previously created one.
288    fn placeholder_region(
289        &mut self,
290        infcx: &InferCtxt<'tcx>,
291        placeholder: ty::PlaceholderRegion,
292    ) -> ty::Region<'tcx> {
293        let placeholder_index = self.placeholder_indices.insert(placeholder);
294        match self.placeholder_index_to_region.get(placeholder_index) {
295            Some(&v) => v,
296            None => {
297                let origin = NllRegionVariableOrigin::Placeholder(placeholder);
298                let region = infcx.next_nll_region_var_in_universe(origin, placeholder.universe);
299                self.placeholder_index_to_region.push(region);
300                region
301            }
302        }
303    }
304}
305
306/// The `Locations` type summarizes *where* region constraints are
307/// required to hold. Normally, this is at a particular point which
308/// created the obligation, but for constraints that the user gave, we
309/// want the constraint to hold at all points.
310#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
311pub enum Locations {
312    /// Indicates that a type constraint should always be true. This
313    /// is particularly important in the new borrowck analysis for
314    /// things like the type of the return slot. Consider this
315    /// example:
316    ///
317    /// ```compile_fail,E0515
318    /// fn foo<'a>(x: &'a u32) -> &'a u32 {
319    ///     let y = 22;
320    ///     return &y; // error
321    /// }
322    /// ```
323    ///
324    /// Here, we wind up with the signature from the return type being
325    /// something like `&'1 u32` where `'1` is a universal region. But
326    /// the type of the return slot `_0` is something like `&'2 u32`
327    /// where `'2` is an existential region variable. The type checker
328    /// requires that `&'2 u32 = &'1 u32` -- but at what point? In the
329    /// older NLL analysis, we required this only at the entry point
330    /// to the function. By the nature of the constraints, this wound
331    /// up propagating to all points reachable from start (because
332    /// `'1` -- as a universal region -- is live everywhere). In the
333    /// newer analysis, though, this doesn't work: `_0` is considered
334    /// dead at the start (it has no usable value) and hence this type
335    /// equality is basically a no-op. Then, later on, when we do `_0
336    /// = &'3 y`, that region `'3` never winds up related to the
337    /// universal region `'1` and hence no error occurs. Therefore, we
338    /// use Locations::All instead, which ensures that the `'1` and
339    /// `'2` are equal everything. We also use this for other
340    /// user-given type annotations; e.g., if the user wrote `let mut
341    /// x: &'static u32 = ...`, we would ensure that all values
342    /// assigned to `x` are of `'static` lifetime.
343    ///
344    /// The span points to the place the constraint arose. For example,
345    /// it points to the type in a user-given type annotation. If
346    /// there's no sensible span then it's DUMMY_SP.
347    All(Span),
348
349    /// An outlives constraint that only has to hold at a single location,
350    /// usually it represents a point where references flow from one spot to
351    /// another (e.g., `x = y`)
352    Single(Location),
353}
354
355impl Locations {
356    pub fn from_location(&self) -> Option<Location> {
357        match self {
358            Locations::All(_) => None,
359            Locations::Single(from_location) => Some(*from_location),
360        }
361    }
362
363    /// Gets a span representing the location.
364    pub fn span(&self, body: &Body<'_>) -> Span {
365        match self {
366            Locations::All(span) => *span,
367            Locations::Single(l) => body.source_info(*l).span,
368        }
369    }
370}
371
372impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
373    fn tcx(&self) -> TyCtxt<'tcx> {
374        self.infcx.tcx
375    }
376
377    fn body(&self) -> &Body<'tcx> {
378        self.body
379    }
380
381    fn to_region_vid(&mut self, r: ty::Region<'tcx>) -> RegionVid {
382        if let ty::RePlaceholder(placeholder) = r.kind() {
383            self.constraints.placeholder_region(self.infcx, placeholder).as_var()
384        } else {
385            self.universal_regions.to_region_vid(r)
386        }
387    }
388
389    fn unsized_feature_enabled(&self) -> bool {
390        let features = self.tcx().features();
391        features.unsized_locals() || features.unsized_fn_params()
392    }
393
394    /// Equate the inferred type and the annotated type for user type annotations
395    #[instrument(skip(self), level = "debug")]
396    fn check_user_type_annotations(&mut self) {
397        debug!(?self.user_type_annotations);
398        let tcx = self.tcx();
399        for user_annotation in self.user_type_annotations {
400            let CanonicalUserTypeAnnotation { span, ref user_ty, inferred_ty } = *user_annotation;
401            let annotation = self.instantiate_canonical(span, user_ty);
402            if let ty::UserTypeKind::TypeOf(def, args) = annotation.kind
403                && let DefKind::InlineConst = tcx.def_kind(def)
404            {
405                assert!(annotation.bounds.is_empty());
406                self.check_inline_const(inferred_ty, def.expect_local(), args, span);
407            } else {
408                self.ascribe_user_type(inferred_ty, annotation, span);
409            }
410        }
411    }
412
413    #[instrument(skip(self, data), level = "debug")]
414    fn push_region_constraints(
415        &mut self,
416        locations: Locations,
417        category: ConstraintCategory<'tcx>,
418        data: &QueryRegionConstraints<'tcx>,
419    ) {
420        debug!("constraints generated: {:#?}", data);
421
422        constraint_conversion::ConstraintConversion::new(
423            self.infcx,
424            self.universal_regions,
425            &self.region_bound_pairs,
426            self.implicit_region_bound,
427            self.infcx.param_env,
428            &self.known_type_outlives_obligations,
429            locations,
430            locations.span(self.body),
431            category,
432            self.constraints,
433        )
434        .convert_all(data);
435    }
436
437    /// Try to relate `sub <: sup`
438    fn sub_types(
439        &mut self,
440        sub: Ty<'tcx>,
441        sup: Ty<'tcx>,
442        locations: Locations,
443        category: ConstraintCategory<'tcx>,
444    ) -> Result<(), NoSolution> {
445        // Use this order of parameters because the sup type is usually the
446        // "expected" type in diagnostics.
447        self.relate_types(sup, ty::Contravariant, sub, locations, category)
448    }
449
450    #[instrument(skip(self, category), level = "debug")]
451    fn eq_types(
452        &mut self,
453        expected: Ty<'tcx>,
454        found: Ty<'tcx>,
455        locations: Locations,
456        category: ConstraintCategory<'tcx>,
457    ) -> Result<(), NoSolution> {
458        self.relate_types(expected, ty::Invariant, found, locations, category)
459    }
460
461    #[instrument(skip(self), level = "debug")]
462    fn relate_type_and_user_type(
463        &mut self,
464        a: Ty<'tcx>,
465        v: ty::Variance,
466        user_ty: &UserTypeProjection,
467        locations: Locations,
468        category: ConstraintCategory<'tcx>,
469    ) -> Result<(), NoSolution> {
470        let annotated_type = self.user_type_annotations[user_ty.base].inferred_ty;
471        trace!(?annotated_type);
472        let mut curr_projected_ty = PlaceTy::from_ty(annotated_type);
473
474        let tcx = self.infcx.tcx;
475
476        for proj in &user_ty.projs {
477            if !self.infcx.next_trait_solver()
478                && let ty::Alias(ty::Opaque, ..) = curr_projected_ty.ty.kind()
479            {
480                // There is nothing that we can compare here if we go through an opaque type.
481                // We're always in its defining scope as we can otherwise not project through
482                // it, so we're constraining it anyways.
483                return Ok(());
484            }
485            let projected_ty = curr_projected_ty.projection_ty_core(
486                tcx,
487                proj,
488                |this, field, ()| {
489                    let ty = this.field_ty(tcx, field);
490                    self.structurally_resolve(ty, locations)
491                },
492                |_, _| unreachable!(),
493            );
494            curr_projected_ty = projected_ty;
495        }
496        trace!(?curr_projected_ty);
497
498        let ty = curr_projected_ty.ty;
499        self.relate_types(ty, v.xform(ty::Contravariant), a, locations, category)?;
500
501        Ok(())
502    }
503
504    fn check_promoted(&mut self, promoted_body: &'a Body<'tcx>, location: Location) {
505        // Determine the constraints from the promoted MIR by running the type
506        // checker on the promoted MIR, then transfer the constraints back to
507        // the main MIR, changing the locations to the provided location.
508
509        let parent_body = mem::replace(&mut self.body, promoted_body);
510
511        // Use new sets of constraints and closure bounds so that we can
512        // modify their locations.
513        let polonius_facts = &mut None;
514        let mut constraints = Default::default();
515        let mut liveness_constraints =
516            LivenessValues::without_specific_points(Rc::new(DenseLocationMap::new(promoted_body)));
517
518        // Don't try to add borrow_region facts for the promoted MIR as they refer
519        // to the wrong locations.
520        let mut swap_constraints = |this: &mut Self| {
521            mem::swap(this.polonius_facts, polonius_facts);
522            mem::swap(&mut this.constraints.outlives_constraints, &mut constraints);
523            mem::swap(&mut this.constraints.liveness_constraints, &mut liveness_constraints);
524        };
525
526        swap_constraints(self);
527
528        self.visit_body(promoted_body);
529
530        self.body = parent_body;
531
532        // Merge the outlives constraints back in, at the given location.
533        swap_constraints(self);
534        let locations = location.to_locations();
535        for constraint in constraints.outlives().iter() {
536            let mut constraint = *constraint;
537            constraint.locations = locations;
538            if let ConstraintCategory::Return(_)
539            | ConstraintCategory::UseAsConst
540            | ConstraintCategory::UseAsStatic = constraint.category
541            {
542                // "Returning" from a promoted is an assignment to a
543                // temporary from the user's point of view.
544                constraint.category = ConstraintCategory::Boring;
545            }
546            self.constraints.outlives_constraints.push(constraint)
547        }
548        // If the region is live at least one location in the promoted MIR,
549        // then add a liveness constraint to the main MIR for this region
550        // at the location provided as an argument to this method
551        //
552        // add_location doesn't care about ordering so not a problem for the live regions to be
553        // unordered.
554        #[allow(rustc::potential_query_instability)]
555        for region in liveness_constraints.live_regions_unordered() {
556            self.constraints.liveness_constraints.add_location(region, location);
557        }
558    }
559
560    fn check_inline_const(
561        &mut self,
562        inferred_ty: Ty<'tcx>,
563        def_id: LocalDefId,
564        args: UserArgs<'tcx>,
565        span: Span,
566    ) {
567        assert!(args.user_self_ty.is_none());
568        let tcx = self.tcx();
569        let const_ty = tcx.type_of(def_id).instantiate(tcx, args.args);
570        if let Err(terr) =
571            self.eq_types(const_ty, inferred_ty, Locations::All(span), ConstraintCategory::Boring)
572        {
573            span_bug!(
574                span,
575                "bad inline const pattern: ({:?} = {:?}) {:?}",
576                const_ty,
577                inferred_ty,
578                terr
579            );
580        }
581        let args = self.infcx.resolve_vars_if_possible(args.args);
582        let predicates = self.prove_closure_bounds(tcx, def_id, args, Locations::All(span));
583        self.normalize_and_prove_instantiated_predicates(
584            def_id.to_def_id(),
585            predicates,
586            Locations::All(span),
587        );
588    }
589}
590
591impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
592    fn visit_span(&mut self, span: Span) {
593        if !span.is_dummy() {
594            debug!(?span);
595            self.last_span = span;
596        }
597    }
598
599    #[instrument(skip(self, body), level = "debug")]
600    fn visit_body(&mut self, body: &Body<'tcx>) {
601        debug_assert!(std::ptr::eq(self.body, body));
602
603        for (local, local_decl) in body.local_decls.iter_enumerated() {
604            self.visit_local_decl(local, local_decl);
605        }
606
607        for (block, block_data) in body.basic_blocks.iter_enumerated() {
608            let mut location = Location { block, statement_index: 0 };
609            for stmt in &block_data.statements {
610                self.visit_statement(stmt, location);
611                location.statement_index += 1;
612            }
613
614            self.visit_terminator(block_data.terminator(), location);
615            self.check_iscleanup(block_data);
616        }
617    }
618
619    #[instrument(skip(self), level = "debug")]
620    fn visit_statement(&mut self, stmt: &Statement<'tcx>, location: Location) {
621        self.super_statement(stmt, location);
622        let tcx = self.tcx();
623        match &stmt.kind {
624            StatementKind::Assign(box (place, rv)) => {
625                // Assignments to temporaries are not "interesting";
626                // they are not caused by the user, but rather artifacts
627                // of lowering. Assignments to other sorts of places *are* interesting
628                // though.
629                let category = match place.as_local() {
630                    Some(RETURN_PLACE) => {
631                        let defining_ty = &self.universal_regions.defining_ty;
632                        if defining_ty.is_const() {
633                            if tcx.is_static(defining_ty.def_id()) {
634                                ConstraintCategory::UseAsStatic
635                            } else {
636                                ConstraintCategory::UseAsConst
637                            }
638                        } else {
639                            ConstraintCategory::Return(ReturnConstraint::Normal)
640                        }
641                    }
642                    Some(l)
643                        if matches!(
644                            self.body.local_decls[l].local_info(),
645                            LocalInfo::AggregateTemp
646                        ) =>
647                    {
648                        ConstraintCategory::Usage
649                    }
650                    Some(l) if !self.body.local_decls[l].is_user_variable() => {
651                        ConstraintCategory::Boring
652                    }
653                    _ => ConstraintCategory::Assignment,
654                };
655                debug!(
656                    "assignment category: {:?} {:?}",
657                    category,
658                    place.as_local().map(|l| &self.body.local_decls[l])
659                );
660
661                let place_ty = place.ty(self.body, tcx).ty;
662                debug!(?place_ty);
663                let place_ty = self.normalize(place_ty, location);
664                debug!("place_ty normalized: {:?}", place_ty);
665                let rv_ty = rv.ty(self.body, tcx);
666                debug!(?rv_ty);
667                let rv_ty = self.normalize(rv_ty, location);
668                debug!("normalized rv_ty: {:?}", rv_ty);
669                if let Err(terr) =
670                    self.sub_types(rv_ty, place_ty, location.to_locations(), category)
671                {
672                    span_mirbug!(
673                        self,
674                        stmt,
675                        "bad assignment ({:?} = {:?}): {:?}",
676                        place_ty,
677                        rv_ty,
678                        terr
679                    );
680                }
681
682                if let Some(annotation_index) = self.rvalue_user_ty(rv) {
683                    if let Err(terr) = self.relate_type_and_user_type(
684                        rv_ty,
685                        ty::Invariant,
686                        &UserTypeProjection { base: annotation_index, projs: vec![] },
687                        location.to_locations(),
688                        ConstraintCategory::TypeAnnotation(AnnotationSource::GenericArg),
689                    ) {
690                        let annotation = &self.user_type_annotations[annotation_index];
691                        span_mirbug!(
692                            self,
693                            stmt,
694                            "bad user type on rvalue ({:?} = {:?}): {:?}",
695                            annotation,
696                            rv_ty,
697                            terr
698                        );
699                    }
700                }
701
702                if !self.unsized_feature_enabled() {
703                    let trait_ref = ty::TraitRef::new(
704                        tcx,
705                        tcx.require_lang_item(LangItem::Sized, Some(self.last_span)),
706                        [place_ty],
707                    );
708                    self.prove_trait_ref(
709                        trait_ref,
710                        location.to_locations(),
711                        ConstraintCategory::SizedBound,
712                    );
713                }
714            }
715            StatementKind::AscribeUserType(box (place, projection), variance) => {
716                let place_ty = place.ty(self.body, tcx).ty;
717                if let Err(terr) = self.relate_type_and_user_type(
718                    place_ty,
719                    *variance,
720                    projection,
721                    Locations::All(stmt.source_info.span),
722                    ConstraintCategory::TypeAnnotation(AnnotationSource::Ascription),
723                ) {
724                    let annotation = &self.user_type_annotations[projection.base];
725                    span_mirbug!(
726                        self,
727                        stmt,
728                        "bad type assert ({:?} <: {:?} with projections {:?}): {:?}",
729                        place_ty,
730                        annotation,
731                        projection.projs,
732                        terr
733                    );
734                }
735            }
736            StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(..))
737            | StatementKind::FakeRead(..)
738            | StatementKind::StorageLive(..)
739            | StatementKind::StorageDead(..)
740            | StatementKind::Retag { .. }
741            | StatementKind::Coverage(..)
742            | StatementKind::ConstEvalCounter
743            | StatementKind::PlaceMention(..)
744            | StatementKind::BackwardIncompatibleDropHint { .. }
745            | StatementKind::Nop => {}
746            StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(..))
747            | StatementKind::Deinit(..)
748            | StatementKind::SetDiscriminant { .. } => {
749                bug!("Statement not allowed in this MIR phase")
750            }
751        }
752    }
753
754    #[instrument(skip(self), level = "debug")]
755    fn visit_terminator(&mut self, term: &Terminator<'tcx>, term_location: Location) {
756        self.super_terminator(term, term_location);
757        let tcx = self.tcx();
758        debug!("terminator kind: {:?}", term.kind);
759        match &term.kind {
760            TerminatorKind::Goto { .. }
761            | TerminatorKind::UnwindResume
762            | TerminatorKind::UnwindTerminate(_)
763            | TerminatorKind::Return
764            | TerminatorKind::CoroutineDrop
765            | TerminatorKind::Unreachable
766            | TerminatorKind::Drop { .. }
767            | TerminatorKind::FalseEdge { .. }
768            | TerminatorKind::FalseUnwind { .. }
769            | TerminatorKind::InlineAsm { .. } => {
770                // no checks needed for these
771            }
772
773            TerminatorKind::SwitchInt { discr, .. } => {
774                let switch_ty = discr.ty(self.body, tcx);
775                if !switch_ty.is_integral() && !switch_ty.is_char() && !switch_ty.is_bool() {
776                    span_mirbug!(self, term, "bad SwitchInt discr ty {:?}", switch_ty);
777                }
778                // FIXME: check the values
779            }
780            TerminatorKind::Call { func, args, .. }
781            | TerminatorKind::TailCall { func, args, .. } => {
782                let call_source = match term.kind {
783                    TerminatorKind::Call { call_source, .. } => call_source,
784                    TerminatorKind::TailCall { .. } => CallSource::Normal,
785                    _ => unreachable!(),
786                };
787
788                let func_ty = func.ty(self.body, tcx);
789                debug!("func_ty.kind: {:?}", func_ty.kind());
790
791                let sig = match func_ty.kind() {
792                    ty::FnDef(..) | ty::FnPtr(..) => func_ty.fn_sig(tcx),
793                    _ => {
794                        span_mirbug!(self, term, "call to non-function {:?}", func_ty);
795                        return;
796                    }
797                };
798                let (unnormalized_sig, map) = tcx.instantiate_bound_regions(sig, |br| {
799                    use crate::renumber::RegionCtxt;
800
801                    let region_ctxt_fn = || {
802                        let reg_info = match br.kind {
803                            ty::BoundRegionKind::Anon => sym::anon,
804                            ty::BoundRegionKind::Named(_, name) => name,
805                            ty::BoundRegionKind::ClosureEnv => sym::env,
806                        };
807
808                        RegionCtxt::LateBound(reg_info)
809                    };
810
811                    self.infcx.next_region_var(
812                        BoundRegion(
813                            term.source_info.span,
814                            br.kind,
815                            BoundRegionConversionTime::FnCall,
816                        ),
817                        region_ctxt_fn,
818                    )
819                });
820                debug!(?unnormalized_sig);
821                // IMPORTANT: We have to prove well formed for the function signature before
822                // we normalize it, as otherwise types like `<&'a &'b () as Trait>::Assoc`
823                // get normalized away, causing us to ignore the `'b: 'a` bound used by the function.
824                //
825                // Normalization results in a well formed type if the input is well formed, so we
826                // don't have to check it twice.
827                //
828                // See #91068 for an example.
829                self.prove_predicates(
830                    unnormalized_sig.inputs_and_output.iter().map(|ty| {
831                        ty::Binder::dummy(ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(
832                            ty.into(),
833                        )))
834                    }),
835                    term_location.to_locations(),
836                    ConstraintCategory::Boring,
837                );
838
839                let sig = self.deeply_normalize(unnormalized_sig, term_location);
840                // HACK(#114936): `WF(sig)` does not imply `WF(normalized(sig))`
841                // with built-in `Fn` implementations, since the impl may not be
842                // well-formed itself.
843                if sig != unnormalized_sig {
844                    self.prove_predicates(
845                        sig.inputs_and_output.iter().map(|ty| {
846                            ty::Binder::dummy(ty::PredicateKind::Clause(
847                                ty::ClauseKind::WellFormed(ty.into()),
848                            ))
849                        }),
850                        term_location.to_locations(),
851                        ConstraintCategory::Boring,
852                    );
853                }
854
855                if let TerminatorKind::Call { destination, target, .. } = term.kind {
856                    self.check_call_dest(term, &sig, destination, target, term_location);
857                }
858
859                // The ordinary liveness rules will ensure that all
860                // regions in the type of the callee are live here. We
861                // then further constrain the late-bound regions that
862                // were instantiated at the call site to be live as
863                // well. The resulting is that all the input (and
864                // output) types in the signature must be live, since
865                // all the inputs that fed into it were live.
866                for &late_bound_region in map.values() {
867                    let region_vid = self.universal_regions.to_region_vid(late_bound_region);
868                    self.constraints.liveness_constraints.add_location(region_vid, term_location);
869                }
870
871                self.check_call_inputs(term, func, &sig, args, term_location, call_source);
872            }
873            TerminatorKind::Assert { cond, msg, .. } => {
874                let cond_ty = cond.ty(self.body, tcx);
875                if cond_ty != tcx.types.bool {
876                    span_mirbug!(self, term, "bad Assert ({:?}, not bool", cond_ty);
877                }
878
879                if let AssertKind::BoundsCheck { len, index } = &**msg {
880                    if len.ty(self.body, tcx) != tcx.types.usize {
881                        span_mirbug!(self, len, "bounds-check length non-usize {:?}", len)
882                    }
883                    if index.ty(self.body, tcx) != tcx.types.usize {
884                        span_mirbug!(self, index, "bounds-check index non-usize {:?}", index)
885                    }
886                }
887            }
888            TerminatorKind::Yield { value, resume_arg, .. } => {
889                match self.body.yield_ty() {
890                    None => span_mirbug!(self, term, "yield in non-coroutine"),
891                    Some(ty) => {
892                        let value_ty = value.ty(self.body, tcx);
893                        if let Err(terr) = self.sub_types(
894                            value_ty,
895                            ty,
896                            term_location.to_locations(),
897                            ConstraintCategory::Yield,
898                        ) {
899                            span_mirbug!(
900                                self,
901                                term,
902                                "type of yield value is {:?}, but the yield type is {:?}: {:?}",
903                                value_ty,
904                                ty,
905                                terr
906                            );
907                        }
908                    }
909                }
910
911                match self.body.resume_ty() {
912                    None => span_mirbug!(self, term, "yield in non-coroutine"),
913                    Some(ty) => {
914                        let resume_ty = resume_arg.ty(self.body, tcx);
915                        if let Err(terr) = self.sub_types(
916                            ty,
917                            resume_ty.ty,
918                            term_location.to_locations(),
919                            ConstraintCategory::Yield,
920                        ) {
921                            span_mirbug!(
922                                self,
923                                term,
924                                "type of resume place is {:?}, but the resume type is {:?}: {:?}",
925                                resume_ty,
926                                ty,
927                                terr
928                            );
929                        }
930                    }
931                }
932            }
933        }
934    }
935
936    fn visit_local_decl(&mut self, local: Local, local_decl: &LocalDecl<'tcx>) {
937        self.super_local_decl(local, local_decl);
938
939        for user_ty in
940            local_decl.user_ty.as_deref().into_iter().flat_map(UserTypeProjections::projections)
941        {
942            let span = self.user_type_annotations[user_ty.base].span;
943
944            let ty = if local_decl.is_nonref_binding() {
945                local_decl.ty
946            } else if let &ty::Ref(_, rty, _) = local_decl.ty.kind() {
947                // If we have a binding of the form `let ref x: T = ..`
948                // then remove the outermost reference so we can check the
949                // type annotation for the remaining type.
950                rty
951            } else {
952                bug!("{:?} with ref binding has wrong type {}", local, local_decl.ty);
953            };
954
955            if let Err(terr) = self.relate_type_and_user_type(
956                ty,
957                ty::Invariant,
958                user_ty,
959                Locations::All(span),
960                ConstraintCategory::TypeAnnotation(AnnotationSource::Declaration),
961            ) {
962                span_mirbug!(
963                    self,
964                    local,
965                    "bad user type on variable {:?}: {:?} != {:?} ({:?})",
966                    local,
967                    local_decl.ty,
968                    local_decl.user_ty,
969                    terr,
970                );
971            }
972        }
973
974        // When `unsized_fn_params` or `unsized_locals` is enabled, only function calls
975        // and nullary ops are checked in `check_call_dest`.
976        if !self.unsized_feature_enabled() {
977            match self.body.local_kind(local) {
978                LocalKind::ReturnPointer | LocalKind::Arg => {
979                    // return values of normal functions are required to be
980                    // sized by typeck, but return values of ADT constructors are
981                    // not because we don't include a `Self: Sized` bounds on them.
982                    //
983                    // Unbound parts of arguments were never required to be Sized
984                    // - maybe we should make that a warning.
985                    return;
986                }
987                LocalKind::Temp => {
988                    let span = local_decl.source_info.span;
989                    let ty = local_decl.ty;
990                    self.ensure_place_sized(ty, span);
991                }
992            }
993        }
994    }
995
996    #[instrument(skip(self), level = "debug")]
997    fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) {
998        self.super_rvalue(rvalue, location);
999        let tcx = self.tcx();
1000        let span = self.body.source_info(location).span;
1001        match rvalue {
1002            Rvalue::Aggregate(ak, ops) => self.check_aggregate_rvalue(rvalue, ak, ops, location),
1003
1004            Rvalue::Repeat(operand, len) => {
1005                let array_ty = rvalue.ty(self.body.local_decls(), tcx);
1006                self.prove_predicate(
1007                    ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(array_ty.into())),
1008                    Locations::Single(location),
1009                    ConstraintCategory::Boring,
1010                );
1011
1012                // If the length cannot be evaluated we must assume that the length can be larger
1013                // than 1.
1014                // If the length is larger than 1, the repeat expression will need to copy the
1015                // element, so we require the `Copy` trait.
1016                if len.try_to_target_usize(tcx).is_none_or(|len| len > 1) {
1017                    match operand {
1018                        Operand::Copy(..) | Operand::Constant(..) => {
1019                            // These are always okay: direct use of a const, or a value that can
1020                            // evidently be copied.
1021                        }
1022                        Operand::Move(place) => {
1023                            // Make sure that repeated elements implement `Copy`.
1024                            let ty = place.ty(self.body, tcx).ty;
1025                            let trait_ref = ty::TraitRef::new(
1026                                tcx,
1027                                tcx.require_lang_item(LangItem::Copy, Some(span)),
1028                                [ty],
1029                            );
1030
1031                            self.prove_trait_ref(
1032                                trait_ref,
1033                                Locations::Single(location),
1034                                ConstraintCategory::CopyBound,
1035                            );
1036                        }
1037                    }
1038                }
1039            }
1040
1041            &Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, ty) => {
1042                let trait_ref = ty::TraitRef::new(
1043                    tcx,
1044                    tcx.require_lang_item(LangItem::Sized, Some(span)),
1045                    [ty],
1046                );
1047
1048                self.prove_trait_ref(
1049                    trait_ref,
1050                    location.to_locations(),
1051                    ConstraintCategory::SizedBound,
1052                );
1053            }
1054            &Rvalue::NullaryOp(NullOp::ContractChecks, _) => {}
1055            &Rvalue::NullaryOp(NullOp::UbChecks, _) => {}
1056
1057            Rvalue::ShallowInitBox(_operand, ty) => {
1058                let trait_ref = ty::TraitRef::new(
1059                    tcx,
1060                    tcx.require_lang_item(LangItem::Sized, Some(span)),
1061                    [*ty],
1062                );
1063
1064                self.prove_trait_ref(
1065                    trait_ref,
1066                    location.to_locations(),
1067                    ConstraintCategory::SizedBound,
1068                );
1069            }
1070
1071            Rvalue::Cast(cast_kind, op, ty) => {
1072                match *cast_kind {
1073                    CastKind::PointerCoercion(PointerCoercion::ReifyFnPointer, coercion_source) => {
1074                        let is_implicit_coercion = coercion_source == CoercionSource::Implicit;
1075                        let src_ty = op.ty(self.body, tcx);
1076                        let mut src_sig = src_ty.fn_sig(tcx);
1077                        if let ty::FnDef(def_id, _) = src_ty.kind()
1078                            && let ty::FnPtr(_, target_hdr) = *ty.kind()
1079                            && tcx.codegen_fn_attrs(def_id).safe_target_features
1080                            && target_hdr.safety.is_safe()
1081                            && let Some(safe_sig) = tcx.adjust_target_feature_sig(
1082                                *def_id,
1083                                src_sig,
1084                                self.body.source.def_id(),
1085                            )
1086                        {
1087                            src_sig = safe_sig;
1088                        }
1089
1090                        // HACK: This shouldn't be necessary... We can remove this when we actually
1091                        // get binders with where clauses, then elaborate implied bounds into that
1092                        // binder, and implement a higher-ranked subtyping algorithm that actually
1093                        // respects these implied bounds.
1094                        //
1095                        // This protects against the case where we are casting from a higher-ranked
1096                        // fn item to a non-higher-ranked fn pointer, where the cast throws away
1097                        // implied bounds that would've needed to be checked at the call site. This
1098                        // only works when we're casting to a non-higher-ranked fn ptr, since
1099                        // placeholders in the target signature could have untracked implied
1100                        // bounds, resulting in incorrect errors.
1101                        //
1102                        // We check that this signature is WF before subtyping the signature with
1103                        // the target fn sig.
1104                        if src_sig.has_bound_regions()
1105                            && let ty::FnPtr(target_fn_tys, target_hdr) = *ty.kind()
1106                            && let target_sig = target_fn_tys.with(target_hdr)
1107                            && let Some(target_sig) = target_sig.no_bound_vars()
1108                        {
1109                            let src_sig = self.infcx.instantiate_binder_with_fresh_vars(
1110                                span,
1111                                BoundRegionConversionTime::HigherRankedType,
1112                                src_sig,
1113                            );
1114                            let src_ty = Ty::new_fn_ptr(self.tcx(), ty::Binder::dummy(src_sig));
1115                            self.prove_predicate(
1116                                ty::ClauseKind::WellFormed(src_ty.into()),
1117                                location.to_locations(),
1118                                ConstraintCategory::Cast { is_implicit_coercion, unsize_to: None },
1119                            );
1120
1121                            let src_ty = self.normalize(src_ty, location);
1122                            if let Err(terr) = self.sub_types(
1123                                src_ty,
1124                                *ty,
1125                                location.to_locations(),
1126                                ConstraintCategory::Cast { is_implicit_coercion, unsize_to: None },
1127                            ) {
1128                                span_mirbug!(
1129                                    self,
1130                                    rvalue,
1131                                    "equating {:?} with {:?} yields {:?}",
1132                                    target_sig,
1133                                    src_sig,
1134                                    terr
1135                                );
1136                            };
1137                        }
1138
1139                        let src_ty = Ty::new_fn_ptr(tcx, src_sig);
1140                        // HACK: We want to assert that the signature of the source fn is
1141                        // well-formed, because we don't enforce that via the WF of FnDef
1142                        // types normally. This should be removed when we improve the tracking
1143                        // of implied bounds of fn signatures.
1144                        self.prove_predicate(
1145                            ty::ClauseKind::WellFormed(src_ty.into()),
1146                            location.to_locations(),
1147                            ConstraintCategory::Cast { is_implicit_coercion, unsize_to: None },
1148                        );
1149
1150                        // The type that we see in the fcx is like
1151                        // `foo::<'a, 'b>`, where `foo` is the path to a
1152                        // function definition. When we extract the
1153                        // signature, it comes from the `fn_sig` query,
1154                        // and hence may contain unnormalized results.
1155                        let src_ty = self.normalize(src_ty, location);
1156                        if let Err(terr) = self.sub_types(
1157                            src_ty,
1158                            *ty,
1159                            location.to_locations(),
1160                            ConstraintCategory::Cast { is_implicit_coercion, unsize_to: None },
1161                        ) {
1162                            span_mirbug!(
1163                                self,
1164                                rvalue,
1165                                "equating {:?} with {:?} yields {:?}",
1166                                src_ty,
1167                                ty,
1168                                terr
1169                            );
1170                        }
1171                    }
1172
1173                    CastKind::PointerCoercion(
1174                        PointerCoercion::ClosureFnPointer(safety),
1175                        coercion_source,
1176                    ) => {
1177                        let sig = match op.ty(self.body, tcx).kind() {
1178                            ty::Closure(_, args) => args.as_closure().sig(),
1179                            _ => bug!(),
1180                        };
1181                        let ty_fn_ptr_from =
1182                            Ty::new_fn_ptr(tcx, tcx.signature_unclosure(sig, safety));
1183
1184                        let is_implicit_coercion = coercion_source == CoercionSource::Implicit;
1185                        if let Err(terr) = self.sub_types(
1186                            ty_fn_ptr_from,
1187                            *ty,
1188                            location.to_locations(),
1189                            ConstraintCategory::Cast { is_implicit_coercion, unsize_to: None },
1190                        ) {
1191                            span_mirbug!(
1192                                self,
1193                                rvalue,
1194                                "equating {:?} with {:?} yields {:?}",
1195                                ty_fn_ptr_from,
1196                                ty,
1197                                terr
1198                            );
1199                        }
1200                    }
1201
1202                    CastKind::PointerCoercion(
1203                        PointerCoercion::UnsafeFnPointer,
1204                        coercion_source,
1205                    ) => {
1206                        let fn_sig = op.ty(self.body, tcx).fn_sig(tcx);
1207
1208                        // The type that we see in the fcx is like
1209                        // `foo::<'a, 'b>`, where `foo` is the path to a
1210                        // function definition. When we extract the
1211                        // signature, it comes from the `fn_sig` query,
1212                        // and hence may contain unnormalized results.
1213                        let fn_sig = self.normalize(fn_sig, location);
1214
1215                        let ty_fn_ptr_from = tcx.safe_to_unsafe_fn_ty(fn_sig);
1216
1217                        let is_implicit_coercion = coercion_source == CoercionSource::Implicit;
1218                        if let Err(terr) = self.sub_types(
1219                            ty_fn_ptr_from,
1220                            *ty,
1221                            location.to_locations(),
1222                            ConstraintCategory::Cast { is_implicit_coercion, unsize_to: None },
1223                        ) {
1224                            span_mirbug!(
1225                                self,
1226                                rvalue,
1227                                "equating {:?} with {:?} yields {:?}",
1228                                ty_fn_ptr_from,
1229                                ty,
1230                                terr
1231                            );
1232                        }
1233                    }
1234
1235                    CastKind::PointerCoercion(PointerCoercion::Unsize, coercion_source) => {
1236                        let &ty = ty;
1237                        let trait_ref = ty::TraitRef::new(
1238                            tcx,
1239                            tcx.require_lang_item(LangItem::CoerceUnsized, Some(span)),
1240                            [op.ty(self.body, tcx), ty],
1241                        );
1242
1243                        let is_implicit_coercion = coercion_source == CoercionSource::Implicit;
1244                        let unsize_to = fold_regions(tcx, ty, |r, _| {
1245                            if let ty::ReVar(_) = r.kind() { tcx.lifetimes.re_erased } else { r }
1246                        });
1247                        self.prove_trait_ref(
1248                            trait_ref,
1249                            location.to_locations(),
1250                            ConstraintCategory::Cast {
1251                                is_implicit_coercion,
1252                                unsize_to: Some(unsize_to),
1253                            },
1254                        );
1255                    }
1256
1257                    CastKind::PointerCoercion(PointerCoercion::DynStar, coercion_source) => {
1258                        // get the constraints from the target type (`dyn* Clone`)
1259                        //
1260                        // apply them to prove that the source type `Foo` implements `Clone` etc
1261                        let (existential_predicates, region) = match ty.kind() {
1262                            Dynamic(predicates, region, ty::DynStar) => (predicates, region),
1263                            _ => panic!("Invalid dyn* cast_ty"),
1264                        };
1265
1266                        let self_ty = op.ty(self.body, tcx);
1267
1268                        let is_implicit_coercion = coercion_source == CoercionSource::Implicit;
1269                        self.prove_predicates(
1270                            existential_predicates
1271                                .iter()
1272                                .map(|predicate| predicate.with_self_ty(tcx, self_ty)),
1273                            location.to_locations(),
1274                            ConstraintCategory::Cast { is_implicit_coercion, unsize_to: None },
1275                        );
1276
1277                        let outlives_predicate = tcx.mk_predicate(Binder::dummy(
1278                            ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(
1279                                ty::OutlivesPredicate(self_ty, *region),
1280                            )),
1281                        ));
1282                        self.prove_predicate(
1283                            outlives_predicate,
1284                            location.to_locations(),
1285                            ConstraintCategory::Cast { is_implicit_coercion, unsize_to: None },
1286                        );
1287                    }
1288
1289                    CastKind::PointerCoercion(
1290                        PointerCoercion::MutToConstPointer,
1291                        coercion_source,
1292                    ) => {
1293                        let ty::RawPtr(ty_from, hir::Mutability::Mut) =
1294                            op.ty(self.body, tcx).kind()
1295                        else {
1296                            span_mirbug!(self, rvalue, "unexpected base type for cast {:?}", ty,);
1297                            return;
1298                        };
1299                        let ty::RawPtr(ty_to, hir::Mutability::Not) = ty.kind() else {
1300                            span_mirbug!(self, rvalue, "unexpected target type for cast {:?}", ty,);
1301                            return;
1302                        };
1303                        let is_implicit_coercion = coercion_source == CoercionSource::Implicit;
1304                        if let Err(terr) = self.sub_types(
1305                            *ty_from,
1306                            *ty_to,
1307                            location.to_locations(),
1308                            ConstraintCategory::Cast { is_implicit_coercion, unsize_to: None },
1309                        ) {
1310                            span_mirbug!(
1311                                self,
1312                                rvalue,
1313                                "relating {:?} with {:?} yields {:?}",
1314                                ty_from,
1315                                ty_to,
1316                                terr
1317                            );
1318                        }
1319                    }
1320
1321                    CastKind::PointerCoercion(PointerCoercion::ArrayToPointer, coercion_source) => {
1322                        let ty_from = op.ty(self.body, tcx);
1323
1324                        let opt_ty_elem_mut = match ty_from.kind() {
1325                            ty::RawPtr(array_ty, array_mut) => match array_ty.kind() {
1326                                ty::Array(ty_elem, _) => Some((ty_elem, *array_mut)),
1327                                _ => None,
1328                            },
1329                            _ => None,
1330                        };
1331
1332                        let Some((ty_elem, ty_mut)) = opt_ty_elem_mut else {
1333                            span_mirbug!(
1334                                self,
1335                                rvalue,
1336                                "ArrayToPointer cast from unexpected type {:?}",
1337                                ty_from,
1338                            );
1339                            return;
1340                        };
1341
1342                        let (ty_to, ty_to_mut) = match ty.kind() {
1343                            ty::RawPtr(ty_to, ty_to_mut) => (ty_to, *ty_to_mut),
1344                            _ => {
1345                                span_mirbug!(
1346                                    self,
1347                                    rvalue,
1348                                    "ArrayToPointer cast to unexpected type {:?}",
1349                                    ty,
1350                                );
1351                                return;
1352                            }
1353                        };
1354
1355                        if ty_to_mut.is_mut() && ty_mut.is_not() {
1356                            span_mirbug!(
1357                                self,
1358                                rvalue,
1359                                "ArrayToPointer cast from const {:?} to mut {:?}",
1360                                ty,
1361                                ty_to
1362                            );
1363                            return;
1364                        }
1365
1366                        let is_implicit_coercion = coercion_source == CoercionSource::Implicit;
1367                        if let Err(terr) = self.sub_types(
1368                            *ty_elem,
1369                            *ty_to,
1370                            location.to_locations(),
1371                            ConstraintCategory::Cast { is_implicit_coercion, unsize_to: None },
1372                        ) {
1373                            span_mirbug!(
1374                                self,
1375                                rvalue,
1376                                "relating {:?} with {:?} yields {:?}",
1377                                ty_elem,
1378                                ty_to,
1379                                terr
1380                            )
1381                        }
1382                    }
1383
1384                    CastKind::PointerExposeProvenance => {
1385                        let ty_from = op.ty(self.body, tcx);
1386                        let cast_ty_from = CastTy::from_ty(ty_from);
1387                        let cast_ty_to = CastTy::from_ty(*ty);
1388                        match (cast_ty_from, cast_ty_to) {
1389                            (Some(CastTy::Ptr(_) | CastTy::FnPtr), Some(CastTy::Int(_))) => (),
1390                            _ => {
1391                                span_mirbug!(
1392                                    self,
1393                                    rvalue,
1394                                    "Invalid PointerExposeProvenance cast {:?} -> {:?}",
1395                                    ty_from,
1396                                    ty
1397                                )
1398                            }
1399                        }
1400                    }
1401
1402                    CastKind::PointerWithExposedProvenance => {
1403                        let ty_from = op.ty(self.body, tcx);
1404                        let cast_ty_from = CastTy::from_ty(ty_from);
1405                        let cast_ty_to = CastTy::from_ty(*ty);
1406                        match (cast_ty_from, cast_ty_to) {
1407                            (Some(CastTy::Int(_)), Some(CastTy::Ptr(_))) => (),
1408                            _ => {
1409                                span_mirbug!(
1410                                    self,
1411                                    rvalue,
1412                                    "Invalid PointerWithExposedProvenance cast {:?} -> {:?}",
1413                                    ty_from,
1414                                    ty
1415                                )
1416                            }
1417                        }
1418                    }
1419                    CastKind::IntToInt => {
1420                        let ty_from = op.ty(self.body, tcx);
1421                        let cast_ty_from = CastTy::from_ty(ty_from);
1422                        let cast_ty_to = CastTy::from_ty(*ty);
1423                        match (cast_ty_from, cast_ty_to) {
1424                            (Some(CastTy::Int(_)), Some(CastTy::Int(_))) => (),
1425                            _ => {
1426                                span_mirbug!(
1427                                    self,
1428                                    rvalue,
1429                                    "Invalid IntToInt cast {:?} -> {:?}",
1430                                    ty_from,
1431                                    ty
1432                                )
1433                            }
1434                        }
1435                    }
1436                    CastKind::IntToFloat => {
1437                        let ty_from = op.ty(self.body, tcx);
1438                        let cast_ty_from = CastTy::from_ty(ty_from);
1439                        let cast_ty_to = CastTy::from_ty(*ty);
1440                        match (cast_ty_from, cast_ty_to) {
1441                            (Some(CastTy::Int(_)), Some(CastTy::Float)) => (),
1442                            _ => {
1443                                span_mirbug!(
1444                                    self,
1445                                    rvalue,
1446                                    "Invalid IntToFloat cast {:?} -> {:?}",
1447                                    ty_from,
1448                                    ty
1449                                )
1450                            }
1451                        }
1452                    }
1453                    CastKind::FloatToInt => {
1454                        let ty_from = op.ty(self.body, tcx);
1455                        let cast_ty_from = CastTy::from_ty(ty_from);
1456                        let cast_ty_to = CastTy::from_ty(*ty);
1457                        match (cast_ty_from, cast_ty_to) {
1458                            (Some(CastTy::Float), Some(CastTy::Int(_))) => (),
1459                            _ => {
1460                                span_mirbug!(
1461                                    self,
1462                                    rvalue,
1463                                    "Invalid FloatToInt cast {:?} -> {:?}",
1464                                    ty_from,
1465                                    ty
1466                                )
1467                            }
1468                        }
1469                    }
1470                    CastKind::FloatToFloat => {
1471                        let ty_from = op.ty(self.body, tcx);
1472                        let cast_ty_from = CastTy::from_ty(ty_from);
1473                        let cast_ty_to = CastTy::from_ty(*ty);
1474                        match (cast_ty_from, cast_ty_to) {
1475                            (Some(CastTy::Float), Some(CastTy::Float)) => (),
1476                            _ => {
1477                                span_mirbug!(
1478                                    self,
1479                                    rvalue,
1480                                    "Invalid FloatToFloat cast {:?} -> {:?}",
1481                                    ty_from,
1482                                    ty
1483                                )
1484                            }
1485                        }
1486                    }
1487                    CastKind::FnPtrToPtr => {
1488                        let ty_from = op.ty(self.body, tcx);
1489                        let cast_ty_from = CastTy::from_ty(ty_from);
1490                        let cast_ty_to = CastTy::from_ty(*ty);
1491                        match (cast_ty_from, cast_ty_to) {
1492                            (Some(CastTy::FnPtr), Some(CastTy::Ptr(_))) => (),
1493                            _ => {
1494                                span_mirbug!(
1495                                    self,
1496                                    rvalue,
1497                                    "Invalid FnPtrToPtr cast {:?} -> {:?}",
1498                                    ty_from,
1499                                    ty
1500                                )
1501                            }
1502                        }
1503                    }
1504                    CastKind::PtrToPtr => {
1505                        let ty_from = op.ty(self.body, tcx);
1506                        let cast_ty_from = CastTy::from_ty(ty_from);
1507                        let cast_ty_to = CastTy::from_ty(*ty);
1508                        match (cast_ty_from, cast_ty_to) {
1509                            (Some(CastTy::Ptr(src)), Some(CastTy::Ptr(dst))) => {
1510                                let src_tail = self.struct_tail(src.ty, location);
1511                                let dst_tail = self.struct_tail(dst.ty, location);
1512
1513                                // This checks (lifetime part of) vtable validity for pointer casts,
1514                                // which is irrelevant when there are aren't principal traits on
1515                                // both sides (aka only auto traits).
1516                                //
1517                                // Note that other checks (such as denying `dyn Send` -> `dyn
1518                                // Debug`) are in `rustc_hir_typeck`.
1519                                if let ty::Dynamic(src_tty, _src_lt, ty::Dyn) = *src_tail.kind()
1520                                    && let ty::Dynamic(dst_tty, dst_lt, ty::Dyn) = *dst_tail.kind()
1521                                    && src_tty.principal().is_some()
1522                                    && dst_tty.principal().is_some()
1523                                {
1524                                    // Remove auto traits.
1525                                    // Auto trait checks are handled in `rustc_hir_typeck` as FCW.
1526                                    let src_obj = Ty::new_dynamic(
1527                                        tcx,
1528                                        tcx.mk_poly_existential_predicates(
1529                                            &src_tty.without_auto_traits().collect::<Vec<_>>(),
1530                                        ),
1531                                        // FIXME: Once we disallow casting `*const dyn Trait + 'short`
1532                                        // to `*const dyn Trait + 'long`, then this can just be `src_lt`.
1533                                        dst_lt,
1534                                        ty::Dyn,
1535                                    );
1536                                    let dst_obj = Ty::new_dynamic(
1537                                        tcx,
1538                                        tcx.mk_poly_existential_predicates(
1539                                            &dst_tty.without_auto_traits().collect::<Vec<_>>(),
1540                                        ),
1541                                        dst_lt,
1542                                        ty::Dyn,
1543                                    );
1544
1545                                    debug!(?src_tty, ?dst_tty, ?src_obj, ?dst_obj);
1546
1547                                    self.sub_types(
1548                                        src_obj,
1549                                        dst_obj,
1550                                        location.to_locations(),
1551                                        ConstraintCategory::Cast {
1552                                            is_implicit_coercion: false,
1553                                            unsize_to: None,
1554                                        },
1555                                    )
1556                                    .unwrap();
1557                                }
1558                            }
1559                            _ => {
1560                                span_mirbug!(
1561                                    self,
1562                                    rvalue,
1563                                    "Invalid PtrToPtr cast {:?} -> {:?}",
1564                                    ty_from,
1565                                    ty
1566                                )
1567                            }
1568                        }
1569                    }
1570                    CastKind::Transmute => {
1571                        span_mirbug!(
1572                            self,
1573                            rvalue,
1574                            "Unexpected CastKind::Transmute, which is not permitted in Analysis MIR",
1575                        );
1576                    }
1577                }
1578            }
1579
1580            Rvalue::Ref(region, _borrow_kind, borrowed_place) => {
1581                self.add_reborrow_constraint(location, *region, borrowed_place);
1582            }
1583
1584            Rvalue::BinaryOp(
1585                BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge,
1586                box (left, right),
1587            ) => {
1588                let ty_left = left.ty(self.body, tcx);
1589                match ty_left.kind() {
1590                    // Types with regions are comparable if they have a common super-type.
1591                    ty::RawPtr(_, _) | ty::FnPtr(..) => {
1592                        let ty_right = right.ty(self.body, tcx);
1593                        let common_ty =
1594                            self.infcx.next_ty_var(self.body.source_info(location).span);
1595                        self.sub_types(
1596                            ty_left,
1597                            common_ty,
1598                            location.to_locations(),
1599                            ConstraintCategory::CallArgument(None),
1600                        )
1601                        .unwrap_or_else(|err| {
1602                            bug!("Could not equate type variable with {:?}: {:?}", ty_left, err)
1603                        });
1604                        if let Err(terr) = self.sub_types(
1605                            ty_right,
1606                            common_ty,
1607                            location.to_locations(),
1608                            ConstraintCategory::CallArgument(None),
1609                        ) {
1610                            span_mirbug!(
1611                                self,
1612                                rvalue,
1613                                "unexpected comparison types {:?} and {:?} yields {:?}",
1614                                ty_left,
1615                                ty_right,
1616                                terr
1617                            )
1618                        }
1619                    }
1620                    // For types with no regions we can just check that the
1621                    // both operands have the same type.
1622                    ty::Int(_) | ty::Uint(_) | ty::Bool | ty::Char | ty::Float(_)
1623                        if ty_left == right.ty(self.body, tcx) => {}
1624                    // Other types are compared by trait methods, not by
1625                    // `Rvalue::BinaryOp`.
1626                    _ => span_mirbug!(
1627                        self,
1628                        rvalue,
1629                        "unexpected comparison types {:?} and {:?}",
1630                        ty_left,
1631                        right.ty(self.body, tcx)
1632                    ),
1633                }
1634            }
1635
1636            Rvalue::WrapUnsafeBinder(op, ty) => {
1637                let operand_ty = op.ty(self.body, self.tcx());
1638                let ty::UnsafeBinder(binder_ty) = *ty.kind() else {
1639                    unreachable!();
1640                };
1641                let expected_ty = self.infcx.instantiate_binder_with_fresh_vars(
1642                    self.body().source_info(location).span,
1643                    BoundRegionConversionTime::HigherRankedType,
1644                    binder_ty.into(),
1645                );
1646                self.sub_types(
1647                    operand_ty,
1648                    expected_ty,
1649                    location.to_locations(),
1650                    ConstraintCategory::Boring,
1651                )
1652                .unwrap();
1653            }
1654
1655            Rvalue::Use(_)
1656            | Rvalue::UnaryOp(_, _)
1657            | Rvalue::CopyForDeref(_)
1658            | Rvalue::BinaryOp(..)
1659            | Rvalue::RawPtr(..)
1660            | Rvalue::ThreadLocalRef(..)
1661            | Rvalue::Len(..)
1662            | Rvalue::Discriminant(..)
1663            | Rvalue::NullaryOp(NullOp::OffsetOf(..), _) => {}
1664        }
1665    }
1666
1667    #[instrument(level = "debug", skip(self))]
1668    fn visit_operand(&mut self, op: &Operand<'tcx>, location: Location) {
1669        self.super_operand(op, location);
1670        if let Operand::Constant(constant) = op {
1671            let maybe_uneval = match constant.const_ {
1672                Const::Val(..) | Const::Ty(_, _) => None,
1673                Const::Unevaluated(uv, _) => Some(uv),
1674            };
1675
1676            if let Some(uv) = maybe_uneval {
1677                if uv.promoted.is_none() {
1678                    let tcx = self.tcx();
1679                    let def_id = uv.def;
1680                    if tcx.def_kind(def_id) == DefKind::InlineConst {
1681                        let def_id = def_id.expect_local();
1682                        let predicates = self.prove_closure_bounds(
1683                            tcx,
1684                            def_id,
1685                            uv.args,
1686                            location.to_locations(),
1687                        );
1688                        self.normalize_and_prove_instantiated_predicates(
1689                            def_id.to_def_id(),
1690                            predicates,
1691                            location.to_locations(),
1692                        );
1693                    }
1694                }
1695            }
1696        }
1697    }
1698
1699    #[instrument(level = "debug", skip(self))]
1700    fn visit_const_operand(&mut self, constant: &ConstOperand<'tcx>, location: Location) {
1701        self.super_const_operand(constant, location);
1702        let ty = constant.const_.ty();
1703
1704        self.infcx.tcx.for_each_free_region(&ty, |live_region| {
1705            let live_region_vid = self.universal_regions.to_region_vid(live_region);
1706            self.constraints.liveness_constraints.add_location(live_region_vid, location);
1707        });
1708
1709        let locations = location.to_locations();
1710        if let Some(annotation_index) = constant.user_ty {
1711            if let Err(terr) = self.relate_type_and_user_type(
1712                constant.const_.ty(),
1713                ty::Invariant,
1714                &UserTypeProjection { base: annotation_index, projs: vec![] },
1715                locations,
1716                ConstraintCategory::TypeAnnotation(AnnotationSource::GenericArg),
1717            ) {
1718                let annotation = &self.user_type_annotations[annotation_index];
1719                span_mirbug!(
1720                    self,
1721                    constant,
1722                    "bad constant user type {:?} vs {:?}: {:?}",
1723                    annotation,
1724                    constant.const_.ty(),
1725                    terr,
1726                );
1727            }
1728        } else {
1729            let tcx = self.tcx();
1730            let maybe_uneval = match constant.const_ {
1731                Const::Ty(_, ct) => match ct.kind() {
1732                    ty::ConstKind::Unevaluated(uv) => {
1733                        Some(UnevaluatedConst { def: uv.def, args: uv.args, promoted: None })
1734                    }
1735                    _ => None,
1736                },
1737                Const::Unevaluated(uv, _) => Some(uv),
1738                _ => None,
1739            };
1740
1741            if let Some(uv) = maybe_uneval {
1742                if let Some(promoted) = uv.promoted {
1743                    let promoted_body = &self.promoted[promoted];
1744                    self.check_promoted(promoted_body, location);
1745                    let promoted_ty = promoted_body.return_ty();
1746                    if let Err(terr) =
1747                        self.eq_types(ty, promoted_ty, locations, ConstraintCategory::Boring)
1748                    {
1749                        span_mirbug!(
1750                            self,
1751                            promoted,
1752                            "bad promoted type ({:?}: {:?}): {:?}",
1753                            ty,
1754                            promoted_ty,
1755                            terr
1756                        );
1757                    };
1758                } else {
1759                    self.ascribe_user_type(
1760                        constant.const_.ty(),
1761                        ty::UserType::new(ty::UserTypeKind::TypeOf(
1762                            uv.def,
1763                            UserArgs { args: uv.args, user_self_ty: None },
1764                        )),
1765                        locations.span(self.body),
1766                    );
1767                }
1768            } else if let Some(static_def_id) = constant.check_static_ptr(tcx) {
1769                let unnormalized_ty = tcx.type_of(static_def_id).instantiate_identity();
1770                let normalized_ty = self.normalize(unnormalized_ty, locations);
1771                let literal_ty = constant.const_.ty().builtin_deref(true).unwrap();
1772
1773                if let Err(terr) =
1774                    self.eq_types(literal_ty, normalized_ty, locations, ConstraintCategory::Boring)
1775                {
1776                    span_mirbug!(self, constant, "bad static type {:?} ({:?})", constant, terr);
1777                }
1778            } else if let Const::Ty(_, ct) = constant.const_
1779                && let ty::ConstKind::Param(p) = ct.kind()
1780            {
1781                let body_def_id = self.universal_regions.defining_ty.def_id();
1782                let const_param = tcx.generics_of(body_def_id).const_param(p, tcx);
1783                self.ascribe_user_type(
1784                    constant.const_.ty(),
1785                    ty::UserType::new(ty::UserTypeKind::TypeOf(
1786                        const_param.def_id,
1787                        UserArgs {
1788                            args: self.universal_regions.defining_ty.args(),
1789                            user_self_ty: None,
1790                        },
1791                    )),
1792                    locations.span(self.body),
1793                );
1794            }
1795
1796            if let ty::FnDef(def_id, args) = *constant.const_.ty().kind() {
1797                let instantiated_predicates = tcx.predicates_of(def_id).instantiate(tcx, args);
1798                self.normalize_and_prove_instantiated_predicates(
1799                    def_id,
1800                    instantiated_predicates,
1801                    locations,
1802                );
1803
1804                assert!(!matches!(
1805                    tcx.impl_of_method(def_id).map(|imp| tcx.def_kind(imp)),
1806                    Some(DefKind::Impl { of_trait: true })
1807                ));
1808                self.prove_predicates(
1809                    args.types().map(|ty| ty::ClauseKind::WellFormed(ty.into())),
1810                    locations,
1811                    ConstraintCategory::Boring,
1812                );
1813            }
1814        }
1815    }
1816
1817    fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) {
1818        self.super_place(place, context, location);
1819        let tcx = self.tcx();
1820        let place_ty = place.ty(self.body, tcx);
1821        if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) = context {
1822            let trait_ref = ty::TraitRef::new(
1823                tcx,
1824                tcx.require_lang_item(LangItem::Copy, Some(self.last_span)),
1825                [place_ty.ty],
1826            );
1827
1828            // To have a `Copy` operand, the type `T` of the
1829            // value must be `Copy`. Note that we prove that `T: Copy`,
1830            // rather than using the `is_copy_modulo_regions`
1831            // test. This is important because
1832            // `is_copy_modulo_regions` ignores the resulting region
1833            // obligations and assumes they pass. This can result in
1834            // bounds from `Copy` impls being unsoundly ignored (e.g.,
1835            // #29149). Note that we decide to use `Copy` before knowing
1836            // whether the bounds fully apply: in effect, the rule is
1837            // that if a value of some type could implement `Copy`, then
1838            // it must.
1839            self.prove_trait_ref(trait_ref, location.to_locations(), ConstraintCategory::CopyBound);
1840        }
1841    }
1842
1843    fn visit_projection_elem(
1844        &mut self,
1845        place: PlaceRef<'tcx>,
1846        elem: PlaceElem<'tcx>,
1847        context: PlaceContext,
1848        location: Location,
1849    ) {
1850        let tcx = self.tcx();
1851        let base_ty = place.ty(self.body(), tcx);
1852        match elem {
1853            // All these projections don't add any constraints, so there's nothing to
1854            // do here. We check their invariants in the MIR validator after all.
1855            ProjectionElem::Deref
1856            | ProjectionElem::Index(_)
1857            | ProjectionElem::ConstantIndex { .. }
1858            | ProjectionElem::Subslice { .. }
1859            | ProjectionElem::Downcast(..) => {}
1860            ProjectionElem::Field(field, fty) => {
1861                let fty = self.normalize(fty, location);
1862                let ty = base_ty.field_ty(tcx, field);
1863                let ty = self.normalize(ty, location);
1864                debug!(?fty, ?ty);
1865
1866                if let Err(terr) = self.relate_types(
1867                    ty,
1868                    context.ambient_variance(),
1869                    fty,
1870                    location.to_locations(),
1871                    ConstraintCategory::Boring,
1872                ) {
1873                    span_mirbug!(self, place, "bad field access ({:?}: {:?}): {:?}", ty, fty, terr);
1874                }
1875            }
1876            ProjectionElem::OpaqueCast(ty) => {
1877                let ty = self.normalize(ty, location);
1878                self.relate_types(
1879                    ty,
1880                    context.ambient_variance(),
1881                    base_ty.ty,
1882                    location.to_locations(),
1883                    ConstraintCategory::TypeAnnotation(AnnotationSource::OpaqueCast),
1884                )
1885                .unwrap();
1886            }
1887            ProjectionElem::UnwrapUnsafeBinder(ty) => {
1888                let ty::UnsafeBinder(binder_ty) = *base_ty.ty.kind() else {
1889                    unreachable!();
1890                };
1891                let found_ty = self.infcx.instantiate_binder_with_fresh_vars(
1892                    self.body.source_info(location).span,
1893                    BoundRegionConversionTime::HigherRankedType,
1894                    binder_ty.into(),
1895                );
1896                self.relate_types(
1897                    ty,
1898                    context.ambient_variance(),
1899                    found_ty,
1900                    location.to_locations(),
1901                    ConstraintCategory::Boring,
1902                )
1903                .unwrap();
1904            }
1905            ProjectionElem::Subtype(_) => {
1906                bug!("ProjectionElem::Subtype shouldn't exist in borrowck")
1907            }
1908        }
1909    }
1910}
1911
1912impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
1913    fn check_call_dest(
1914        &mut self,
1915        term: &Terminator<'tcx>,
1916        sig: &ty::FnSig<'tcx>,
1917        destination: Place<'tcx>,
1918        target: Option<BasicBlock>,
1919        term_location: Location,
1920    ) {
1921        let tcx = self.tcx();
1922        match target {
1923            Some(_) => {
1924                let dest_ty = destination.ty(self.body, tcx).ty;
1925                let dest_ty = self.normalize(dest_ty, term_location);
1926                let category = match destination.as_local() {
1927                    Some(RETURN_PLACE) => {
1928                        if let DefiningTy::Const(def_id, _) | DefiningTy::InlineConst(def_id, _) =
1929                            self.universal_regions.defining_ty
1930                        {
1931                            if tcx.is_static(def_id) {
1932                                ConstraintCategory::UseAsStatic
1933                            } else {
1934                                ConstraintCategory::UseAsConst
1935                            }
1936                        } else {
1937                            ConstraintCategory::Return(ReturnConstraint::Normal)
1938                        }
1939                    }
1940                    Some(l) if !self.body.local_decls[l].is_user_variable() => {
1941                        ConstraintCategory::Boring
1942                    }
1943                    // The return type of a call is interesting for diagnostics.
1944                    _ => ConstraintCategory::Assignment,
1945                };
1946
1947                let locations = term_location.to_locations();
1948
1949                if let Err(terr) = self.sub_types(sig.output(), dest_ty, locations, category) {
1950                    span_mirbug!(
1951                        self,
1952                        term,
1953                        "call dest mismatch ({:?} <- {:?}): {:?}",
1954                        dest_ty,
1955                        sig.output(),
1956                        terr
1957                    );
1958                }
1959
1960                // When `unsized_fn_params` and `unsized_locals` are both not enabled,
1961                // this check is done at `check_local`.
1962                if self.unsized_feature_enabled() {
1963                    let span = term.source_info.span;
1964                    self.ensure_place_sized(dest_ty, span);
1965                }
1966            }
1967            None => {
1968                // The signature in this call can reference region variables,
1969                // so erase them before calling a query.
1970                let output_ty = self.tcx().erase_regions(sig.output());
1971                if !output_ty.is_privately_uninhabited(
1972                    self.tcx(),
1973                    self.infcx.typing_env(self.infcx.param_env),
1974                ) {
1975                    span_mirbug!(self, term, "call to converging function {:?} w/o dest", sig);
1976                }
1977            }
1978        }
1979    }
1980
1981    #[instrument(level = "debug", skip(self, term, func, term_location, call_source))]
1982    fn check_call_inputs(
1983        &mut self,
1984        term: &Terminator<'tcx>,
1985        func: &Operand<'tcx>,
1986        sig: &ty::FnSig<'tcx>,
1987        args: &[Spanned<Operand<'tcx>>],
1988        term_location: Location,
1989        call_source: CallSource,
1990    ) {
1991        if args.len() < sig.inputs().len() || (args.len() > sig.inputs().len() && !sig.c_variadic) {
1992            span_mirbug!(self, term, "call to {:?} with wrong # of args", sig);
1993        }
1994
1995        let func_ty = func.ty(self.body, self.infcx.tcx);
1996        if let ty::FnDef(def_id, _) = *func_ty.kind() {
1997            // Some of the SIMD intrinsics are special: they need a particular argument to be a
1998            // constant. (Eventually this should use const-generics, but those are not up for the
1999            // task yet: https://github.com/rust-lang/rust/issues/85229.)
2000            if let Some(name @ (sym::simd_shuffle | sym::simd_insert | sym::simd_extract)) =
2001                self.tcx().intrinsic(def_id).map(|i| i.name)
2002            {
2003                let idx = match name {
2004                    sym::simd_shuffle => 2,
2005                    _ => 1,
2006                };
2007                if !matches!(args[idx], Spanned { node: Operand::Constant(_), .. }) {
2008                    self.tcx().dcx().emit_err(SimdIntrinsicArgConst {
2009                        span: term.source_info.span,
2010                        arg: idx + 1,
2011                        intrinsic: name.to_string(),
2012                    });
2013                }
2014            }
2015        }
2016        debug!(?func_ty);
2017
2018        for (n, (fn_arg, op_arg)) in iter::zip(sig.inputs(), args).enumerate() {
2019            let op_arg_ty = op_arg.node.ty(self.body, self.tcx());
2020
2021            let op_arg_ty = self.normalize(op_arg_ty, term_location);
2022            let category = if call_source.from_hir_call() {
2023                ConstraintCategory::CallArgument(Some(self.infcx.tcx.erase_regions(func_ty)))
2024            } else {
2025                ConstraintCategory::Boring
2026            };
2027            if let Err(terr) =
2028                self.sub_types(op_arg_ty, *fn_arg, term_location.to_locations(), category)
2029            {
2030                span_mirbug!(
2031                    self,
2032                    term,
2033                    "bad arg #{:?} ({:?} <- {:?}): {:?}",
2034                    n,
2035                    fn_arg,
2036                    op_arg_ty,
2037                    terr
2038                );
2039            }
2040        }
2041    }
2042
2043    fn check_iscleanup(&mut self, block_data: &BasicBlockData<'tcx>) {
2044        let is_cleanup = block_data.is_cleanup;
2045        match block_data.terminator().kind {
2046            TerminatorKind::Goto { target } => {
2047                self.assert_iscleanup(block_data, target, is_cleanup)
2048            }
2049            TerminatorKind::SwitchInt { ref targets, .. } => {
2050                for target in targets.all_targets() {
2051                    self.assert_iscleanup(block_data, *target, is_cleanup);
2052                }
2053            }
2054            TerminatorKind::UnwindResume => {
2055                if !is_cleanup {
2056                    span_mirbug!(self, block_data, "resume on non-cleanup block!")
2057                }
2058            }
2059            TerminatorKind::UnwindTerminate(_) => {
2060                if !is_cleanup {
2061                    span_mirbug!(self, block_data, "terminate on non-cleanup block!")
2062                }
2063            }
2064            TerminatorKind::Return => {
2065                if is_cleanup {
2066                    span_mirbug!(self, block_data, "return on cleanup block")
2067                }
2068            }
2069            TerminatorKind::TailCall { .. } => {
2070                if is_cleanup {
2071                    span_mirbug!(self, block_data, "tailcall on cleanup block")
2072                }
2073            }
2074            TerminatorKind::CoroutineDrop { .. } => {
2075                if is_cleanup {
2076                    span_mirbug!(self, block_data, "coroutine_drop in cleanup block")
2077                }
2078            }
2079            TerminatorKind::Yield { resume, drop, .. } => {
2080                if is_cleanup {
2081                    span_mirbug!(self, block_data, "yield in cleanup block")
2082                }
2083                self.assert_iscleanup(block_data, resume, is_cleanup);
2084                if let Some(drop) = drop {
2085                    self.assert_iscleanup(block_data, drop, is_cleanup);
2086                }
2087            }
2088            TerminatorKind::Unreachable => {}
2089            TerminatorKind::Drop { target, unwind, .. }
2090            | TerminatorKind::Assert { target, unwind, .. } => {
2091                self.assert_iscleanup(block_data, target, is_cleanup);
2092                self.assert_iscleanup_unwind(block_data, unwind, is_cleanup);
2093            }
2094            TerminatorKind::Call { ref target, unwind, .. } => {
2095                if let &Some(target) = target {
2096                    self.assert_iscleanup(block_data, target, is_cleanup);
2097                }
2098                self.assert_iscleanup_unwind(block_data, unwind, is_cleanup);
2099            }
2100            TerminatorKind::FalseEdge { real_target, imaginary_target } => {
2101                self.assert_iscleanup(block_data, real_target, is_cleanup);
2102                self.assert_iscleanup(block_data, imaginary_target, is_cleanup);
2103            }
2104            TerminatorKind::FalseUnwind { real_target, unwind } => {
2105                self.assert_iscleanup(block_data, real_target, is_cleanup);
2106                self.assert_iscleanup_unwind(block_data, unwind, is_cleanup);
2107            }
2108            TerminatorKind::InlineAsm { ref targets, unwind, .. } => {
2109                for &target in targets {
2110                    self.assert_iscleanup(block_data, target, is_cleanup);
2111                }
2112                self.assert_iscleanup_unwind(block_data, unwind, is_cleanup);
2113            }
2114        }
2115    }
2116
2117    fn assert_iscleanup(&mut self, ctxt: &dyn fmt::Debug, bb: BasicBlock, iscleanuppad: bool) {
2118        if self.body[bb].is_cleanup != iscleanuppad {
2119            span_mirbug!(self, ctxt, "cleanuppad mismatch: {:?} should be {:?}", bb, iscleanuppad);
2120        }
2121    }
2122
2123    fn assert_iscleanup_unwind(
2124        &mut self,
2125        ctxt: &dyn fmt::Debug,
2126        unwind: UnwindAction,
2127        is_cleanup: bool,
2128    ) {
2129        match unwind {
2130            UnwindAction::Cleanup(unwind) => {
2131                if is_cleanup {
2132                    span_mirbug!(self, ctxt, "unwind on cleanup block")
2133                }
2134                self.assert_iscleanup(ctxt, unwind, true);
2135            }
2136            UnwindAction::Continue => {
2137                if is_cleanup {
2138                    span_mirbug!(self, ctxt, "unwind on cleanup block")
2139                }
2140            }
2141            UnwindAction::Unreachable | UnwindAction::Terminate(_) => (),
2142        }
2143    }
2144
2145    fn ensure_place_sized(&mut self, ty: Ty<'tcx>, span: Span) {
2146        let tcx = self.tcx();
2147
2148        // Erase the regions from `ty` to get a global type. The
2149        // `Sized` bound in no way depends on precise regions, so this
2150        // shouldn't affect `is_sized`.
2151        let erased_ty = tcx.erase_regions(ty);
2152        // FIXME(#132279): Using `Ty::is_sized` causes us to incorrectly handle opaques here.
2153        if !erased_ty.is_sized(tcx, self.infcx.typing_env(self.infcx.param_env)) {
2154            // in current MIR construction, all non-control-flow rvalue
2155            // expressions evaluate through `as_temp` or `into` a return
2156            // slot or local, so to find all unsized rvalues it is enough
2157            // to check all temps, return slots and locals.
2158            if self.reported_errors.replace((ty, span)).is_none() {
2159                // While this is located in `nll::typeck` this error is not
2160                // an NLL error, it's a required check to prevent creation
2161                // of unsized rvalues in a call expression.
2162                self.tcx().dcx().emit_err(MoveUnsized { ty, span });
2163            }
2164        }
2165    }
2166
2167    fn aggregate_field_ty(
2168        &mut self,
2169        ak: &AggregateKind<'tcx>,
2170        field_index: FieldIdx,
2171        location: Location,
2172    ) -> Result<Ty<'tcx>, FieldAccessError> {
2173        let tcx = self.tcx();
2174
2175        match *ak {
2176            AggregateKind::Adt(adt_did, variant_index, args, _, active_field_index) => {
2177                let def = tcx.adt_def(adt_did);
2178                let variant = &def.variant(variant_index);
2179                let adj_field_index = active_field_index.unwrap_or(field_index);
2180                if let Some(field) = variant.fields.get(adj_field_index) {
2181                    Ok(self.normalize(field.ty(tcx, args), location))
2182                } else {
2183                    Err(FieldAccessError::OutOfRange { field_count: variant.fields.len() })
2184                }
2185            }
2186            AggregateKind::Closure(_, args) => {
2187                match args.as_closure().upvar_tys().get(field_index.as_usize()) {
2188                    Some(ty) => Ok(*ty),
2189                    None => Err(FieldAccessError::OutOfRange {
2190                        field_count: args.as_closure().upvar_tys().len(),
2191                    }),
2192                }
2193            }
2194            AggregateKind::Coroutine(_, args) => {
2195                // It doesn't make sense to look at a field beyond the prefix;
2196                // these require a variant index, and are not initialized in
2197                // aggregate rvalues.
2198                match args.as_coroutine().prefix_tys().get(field_index.as_usize()) {
2199                    Some(ty) => Ok(*ty),
2200                    None => Err(FieldAccessError::OutOfRange {
2201                        field_count: args.as_coroutine().prefix_tys().len(),
2202                    }),
2203                }
2204            }
2205            AggregateKind::CoroutineClosure(_, args) => {
2206                match args.as_coroutine_closure().upvar_tys().get(field_index.as_usize()) {
2207                    Some(ty) => Ok(*ty),
2208                    None => Err(FieldAccessError::OutOfRange {
2209                        field_count: args.as_coroutine_closure().upvar_tys().len(),
2210                    }),
2211                }
2212            }
2213            AggregateKind::Array(ty) => Ok(ty),
2214            AggregateKind::Tuple | AggregateKind::RawPtr(..) => {
2215                unreachable!("This should have been covered in check_rvalues");
2216            }
2217        }
2218    }
2219
2220    /// If this rvalue supports a user-given type annotation, then
2221    /// extract and return it. This represents the final type of the
2222    /// rvalue and will be unified with the inferred type.
2223    fn rvalue_user_ty(&self, rvalue: &Rvalue<'tcx>) -> Option<UserTypeAnnotationIndex> {
2224        match rvalue {
2225            Rvalue::Use(_)
2226            | Rvalue::ThreadLocalRef(_)
2227            | Rvalue::Repeat(..)
2228            | Rvalue::Ref(..)
2229            | Rvalue::RawPtr(..)
2230            | Rvalue::Len(..)
2231            | Rvalue::Cast(..)
2232            | Rvalue::ShallowInitBox(..)
2233            | Rvalue::BinaryOp(..)
2234            | Rvalue::NullaryOp(..)
2235            | Rvalue::CopyForDeref(..)
2236            | Rvalue::UnaryOp(..)
2237            | Rvalue::Discriminant(..)
2238            | Rvalue::WrapUnsafeBinder(..) => None,
2239
2240            Rvalue::Aggregate(aggregate, _) => match **aggregate {
2241                AggregateKind::Adt(_, _, _, user_ty, _) => user_ty,
2242                AggregateKind::Array(_) => None,
2243                AggregateKind::Tuple => None,
2244                AggregateKind::Closure(_, _) => None,
2245                AggregateKind::Coroutine(_, _) => None,
2246                AggregateKind::CoroutineClosure(_, _) => None,
2247                AggregateKind::RawPtr(_, _) => None,
2248            },
2249        }
2250    }
2251
2252    fn check_aggregate_rvalue(
2253        &mut self,
2254        rvalue: &Rvalue<'tcx>,
2255        aggregate_kind: &AggregateKind<'tcx>,
2256        operands: &IndexSlice<FieldIdx, Operand<'tcx>>,
2257        location: Location,
2258    ) {
2259        let tcx = self.tcx();
2260
2261        self.prove_aggregate_predicates(aggregate_kind, location);
2262
2263        if *aggregate_kind == AggregateKind::Tuple {
2264            // tuple rvalue field type is always the type of the op. Nothing to check here.
2265            return;
2266        }
2267
2268        if let AggregateKind::RawPtr(..) = aggregate_kind {
2269            bug!("RawPtr should only be in runtime MIR");
2270        }
2271
2272        for (i, operand) in operands.iter_enumerated() {
2273            let field_ty = match self.aggregate_field_ty(aggregate_kind, i, location) {
2274                Ok(field_ty) => field_ty,
2275                Err(FieldAccessError::OutOfRange { field_count }) => {
2276                    span_mirbug!(
2277                        self,
2278                        rvalue,
2279                        "accessed field #{} but variant only has {}",
2280                        i.as_u32(),
2281                        field_count,
2282                    );
2283                    continue;
2284                }
2285            };
2286            let operand_ty = operand.ty(self.body, tcx);
2287            let operand_ty = self.normalize(operand_ty, location);
2288
2289            if let Err(terr) = self.sub_types(
2290                operand_ty,
2291                field_ty,
2292                location.to_locations(),
2293                ConstraintCategory::Boring,
2294            ) {
2295                span_mirbug!(
2296                    self,
2297                    rvalue,
2298                    "{:?} is not a subtype of {:?}: {:?}",
2299                    operand_ty,
2300                    field_ty,
2301                    terr
2302                );
2303            }
2304        }
2305    }
2306
2307    /// Adds the constraints that arise from a borrow expression `&'a P` at the location `L`.
2308    ///
2309    /// # Parameters
2310    ///
2311    /// - `location`: the location `L` where the borrow expression occurs
2312    /// - `borrow_region`: the region `'a` associated with the borrow
2313    /// - `borrowed_place`: the place `P` being borrowed
2314    fn add_reborrow_constraint(
2315        &mut self,
2316        location: Location,
2317        borrow_region: ty::Region<'tcx>,
2318        borrowed_place: &Place<'tcx>,
2319    ) {
2320        // These constraints are only meaningful during borrowck:
2321        let Self { borrow_set, location_table, polonius_facts, constraints, .. } = self;
2322
2323        // In Polonius mode, we also push a `loan_issued_at` fact
2324        // linking the loan to the region (in some cases, though,
2325        // there is no loan associated with this borrow expression --
2326        // that occurs when we are borrowing an unsafe place, for
2327        // example).
2328        if let Some(polonius_facts) = polonius_facts {
2329            let _prof_timer = self.infcx.tcx.prof.generic_activity("polonius_fact_generation");
2330            if let Some(borrow_index) = borrow_set.get_index_of(&location) {
2331                let region_vid = borrow_region.as_var();
2332                polonius_facts.loan_issued_at.push((
2333                    region_vid.into(),
2334                    borrow_index,
2335                    location_table.mid_index(location),
2336                ));
2337            }
2338        }
2339
2340        // If we are reborrowing the referent of another reference, we
2341        // need to add outlives relationships. In a case like `&mut
2342        // *p`, where the `p` has type `&'b mut Foo`, for example, we
2343        // need to ensure that `'b: 'a`.
2344
2345        debug!(
2346            "add_reborrow_constraint({:?}, {:?}, {:?})",
2347            location, borrow_region, borrowed_place
2348        );
2349
2350        let tcx = self.infcx.tcx;
2351        let def = self.body.source.def_id().expect_local();
2352        let upvars = tcx.closure_captures(def);
2353        let field =
2354            path_utils::is_upvar_field_projection(tcx, upvars, borrowed_place.as_ref(), self.body);
2355        let category = if let Some(field) = field {
2356            ConstraintCategory::ClosureUpvar(field)
2357        } else {
2358            ConstraintCategory::Boring
2359        };
2360
2361        for (base, elem) in borrowed_place.as_ref().iter_projections().rev() {
2362            debug!("add_reborrow_constraint - iteration {:?}", elem);
2363
2364            match elem {
2365                ProjectionElem::Deref => {
2366                    let base_ty = base.ty(self.body, tcx).ty;
2367
2368                    debug!("add_reborrow_constraint - base_ty = {:?}", base_ty);
2369                    match base_ty.kind() {
2370                        ty::Ref(ref_region, _, mutbl) => {
2371                            constraints.outlives_constraints.push(OutlivesConstraint {
2372                                sup: ref_region.as_var(),
2373                                sub: borrow_region.as_var(),
2374                                locations: location.to_locations(),
2375                                span: location.to_locations().span(self.body),
2376                                category,
2377                                variance_info: ty::VarianceDiagInfo::default(),
2378                                from_closure: false,
2379                            });
2380
2381                            match mutbl {
2382                                hir::Mutability::Not => {
2383                                    // Immutable reference. We don't need the base
2384                                    // to be valid for the entire lifetime of
2385                                    // the borrow.
2386                                    break;
2387                                }
2388                                hir::Mutability::Mut => {
2389                                    // Mutable reference. We *do* need the base
2390                                    // to be valid, because after the base becomes
2391                                    // invalid, someone else can use our mutable deref.
2392
2393                                    // This is in order to make the following function
2394                                    // illegal:
2395                                    // ```
2396                                    // fn unsafe_deref<'a, 'b>(x: &'a &'b mut T) -> &'b mut T {
2397                                    //     &mut *x
2398                                    // }
2399                                    // ```
2400                                    //
2401                                    // As otherwise you could clone `&mut T` using the
2402                                    // following function:
2403                                    // ```
2404                                    // fn bad(x: &mut T) -> (&mut T, &mut T) {
2405                                    //     let my_clone = unsafe_deref(&'a x);
2406                                    //     ENDREGION 'a;
2407                                    //     (my_clone, x)
2408                                    // }
2409                                    // ```
2410                                }
2411                            }
2412                        }
2413                        ty::RawPtr(..) => {
2414                            // deref of raw pointer, guaranteed to be valid
2415                            break;
2416                        }
2417                        ty::Adt(def, _) if def.is_box() => {
2418                            // deref of `Box`, need the base to be valid - propagate
2419                        }
2420                        _ => bug!("unexpected deref ty {:?} in {:?}", base_ty, borrowed_place),
2421                    }
2422                }
2423                ProjectionElem::Field(..)
2424                | ProjectionElem::Downcast(..)
2425                | ProjectionElem::OpaqueCast(..)
2426                | ProjectionElem::Index(..)
2427                | ProjectionElem::ConstantIndex { .. }
2428                | ProjectionElem::Subslice { .. }
2429                | ProjectionElem::UnwrapUnsafeBinder(_) => {
2430                    // other field access
2431                }
2432                ProjectionElem::Subtype(_) => {
2433                    bug!("ProjectionElem::Subtype shouldn't exist in borrowck")
2434                }
2435            }
2436        }
2437    }
2438
2439    fn prove_aggregate_predicates(
2440        &mut self,
2441        aggregate_kind: &AggregateKind<'tcx>,
2442        location: Location,
2443    ) {
2444        let tcx = self.tcx();
2445
2446        debug!(
2447            "prove_aggregate_predicates(aggregate_kind={:?}, location={:?})",
2448            aggregate_kind, location
2449        );
2450
2451        let (def_id, instantiated_predicates) = match *aggregate_kind {
2452            AggregateKind::Adt(adt_did, _, args, _, _) => {
2453                (adt_did, tcx.predicates_of(adt_did).instantiate(tcx, args))
2454            }
2455
2456            // For closures, we have some **extra requirements** we
2457            // have to check. In particular, in their upvars and
2458            // signatures, closures often reference various regions
2459            // from the surrounding function -- we call those the
2460            // closure's free regions. When we borrow-check (and hence
2461            // region-check) closures, we may find that the closure
2462            // requires certain relationships between those free
2463            // regions. However, because those free regions refer to
2464            // portions of the CFG of their caller, the closure is not
2465            // in a position to verify those relationships. In that
2466            // case, the requirements get "propagated" to us, and so
2467            // we have to solve them here where we instantiate the
2468            // closure.
2469            //
2470            // Despite the opacity of the previous paragraph, this is
2471            // actually relatively easy to understand in terms of the
2472            // desugaring. A closure gets desugared to a struct, and
2473            // these extra requirements are basically like where
2474            // clauses on the struct.
2475            AggregateKind::Closure(def_id, args)
2476            | AggregateKind::CoroutineClosure(def_id, args)
2477            | AggregateKind::Coroutine(def_id, args) => (
2478                def_id,
2479                self.prove_closure_bounds(
2480                    tcx,
2481                    def_id.expect_local(),
2482                    args,
2483                    location.to_locations(),
2484                ),
2485            ),
2486
2487            AggregateKind::Array(_) | AggregateKind::Tuple | AggregateKind::RawPtr(..) => {
2488                (CRATE_DEF_ID.to_def_id(), ty::InstantiatedPredicates::empty())
2489            }
2490        };
2491
2492        self.normalize_and_prove_instantiated_predicates(
2493            def_id,
2494            instantiated_predicates,
2495            location.to_locations(),
2496        );
2497    }
2498
2499    fn prove_closure_bounds(
2500        &mut self,
2501        tcx: TyCtxt<'tcx>,
2502        def_id: LocalDefId,
2503        args: GenericArgsRef<'tcx>,
2504        locations: Locations,
2505    ) -> ty::InstantiatedPredicates<'tcx> {
2506        let closure_borrowck_results = tcx.mir_borrowck(def_id);
2507        self.concrete_opaque_types
2508            .extend_from_nested_body(tcx, &closure_borrowck_results.concrete_opaque_types);
2509
2510        if let Some(closure_requirements) = &closure_borrowck_results.closure_requirements {
2511            constraint_conversion::ConstraintConversion::new(
2512                self.infcx,
2513                self.universal_regions,
2514                &self.region_bound_pairs,
2515                self.implicit_region_bound,
2516                self.infcx.param_env,
2517                &self.known_type_outlives_obligations,
2518                locations,
2519                self.body.span,             // irrelevant; will be overridden.
2520                ConstraintCategory::Boring, // same as above.
2521                self.constraints,
2522            )
2523            .apply_closure_requirements(closure_requirements, def_id, args);
2524        }
2525
2526        // Now equate closure args to regions inherited from `typeck_root_def_id`. Fixes #98589.
2527        let typeck_root_def_id = tcx.typeck_root_def_id(self.body.source.def_id());
2528        let typeck_root_args = ty::GenericArgs::identity_for_item(tcx, typeck_root_def_id);
2529
2530        let parent_args = match tcx.def_kind(def_id) {
2531            // We don't want to dispatch on 3 different kind of closures here, so take
2532            // advantage of the fact that the `parent_args` is the same length as the
2533            // `typeck_root_args`.
2534            DefKind::Closure => {
2535                // FIXME(async_closures): It may be useful to add a debug assert here
2536                // to actually call `type_of` and check the `parent_args` are the same
2537                // length as the `typeck_root_args`.
2538                &args[..typeck_root_args.len()]
2539            }
2540            DefKind::InlineConst => args.as_inline_const().parent_args(),
2541            other => bug!("unexpected item {:?}", other),
2542        };
2543        let parent_args = tcx.mk_args(parent_args);
2544
2545        assert_eq!(typeck_root_args.len(), parent_args.len());
2546        if let Err(_) = self.eq_args(
2547            typeck_root_args,
2548            parent_args,
2549            locations,
2550            ConstraintCategory::BoringNoLocation,
2551        ) {
2552            span_mirbug!(
2553                self,
2554                def_id,
2555                "could not relate closure to parent {:?} != {:?}",
2556                typeck_root_args,
2557                parent_args
2558            );
2559        }
2560
2561        tcx.predicates_of(def_id).instantiate(tcx, args)
2562    }
2563}
2564
2565trait NormalizeLocation: fmt::Debug + Copy {
2566    fn to_locations(self) -> Locations;
2567}
2568
2569impl NormalizeLocation for Locations {
2570    fn to_locations(self) -> Locations {
2571        self
2572    }
2573}
2574
2575impl NormalizeLocation for Location {
2576    fn to_locations(self) -> Locations {
2577        Locations::Single(self)
2578    }
2579}
2580
2581/// Runs `infcx.instantiate_opaque_types`. Unlike other `TypeOp`s,
2582/// this is not canonicalized - it directly affects the main `InferCtxt`
2583/// that we use during MIR borrowchecking.
2584#[derive(Debug)]
2585pub(super) struct InstantiateOpaqueType<'tcx> {
2586    pub base_universe: Option<ty::UniverseIndex>,
2587    pub region_constraints: Option<RegionConstraintData<'tcx>>,
2588    pub obligations: PredicateObligations<'tcx>,
2589}
2590
2591impl<'tcx> TypeOp<'tcx> for InstantiateOpaqueType<'tcx> {
2592    type Output = ();
2593    /// We use this type itself to store the information used
2594    /// when reporting errors. Since this is not a query, we don't
2595    /// re-run anything during error reporting - we just use the information
2596    /// we saved to help extract an error from the already-existing region
2597    /// constraints in our `InferCtxt`
2598    type ErrorInfo = InstantiateOpaqueType<'tcx>;
2599
2600    fn fully_perform(
2601        mut self,
2602        infcx: &InferCtxt<'tcx>,
2603        span: Span,
2604    ) -> Result<TypeOpOutput<'tcx, Self>, ErrorGuaranteed> {
2605        let (mut output, region_constraints) = scrape_region_constraints(
2606            infcx,
2607            |ocx| {
2608                ocx.register_obligations(self.obligations.clone());
2609                Ok(())
2610            },
2611            "InstantiateOpaqueType",
2612            span,
2613        )?;
2614        self.region_constraints = Some(region_constraints);
2615        output.error_info = Some(self);
2616        Ok(output)
2617    }
2618}