diff --git a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs index 743a28822eb97..38aba06eec5b6 100644 --- a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs +++ b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs @@ -653,8 +653,8 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> { // We want to focus on relevant live locals in diagnostics, so when polonius is enabled, we // ensure that we don't emit live boring locals as explanations. let is_local_boring = |local| { - if let Some(polonius_diagnostics) = self.polonius_diagnostics { - polonius_diagnostics.boring_nll_locals.contains(&local) + if let Some(polonius_context) = self.polonius_context { + polonius_context.boring_nll_locals.contains(&local) } else { assert!(!tcx.sess.opts.unstable_opts.polonius.is_next_enabled()); diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index 91defbad0a0e0..961c30b9fa362 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -62,10 +62,10 @@ use crate::diagnostics::{ use crate::path_utils::*; use crate::place_ext::PlaceExt; use crate::places_conflict::{PlaceConflictBias, places_conflict}; +use crate::polonius::PoloniusContext; use crate::polonius::legacy::{ PoloniusFacts, PoloniusFactsExt, PoloniusLocationTable, PoloniusOutput, }; -use crate::polonius::{PoloniusContext, PoloniusDiagnosticsContext}; use crate::prefixes::PrefixSet; use crate::region_infer::RegionInferenceContext; use crate::region_infer::opaque_types::DeferredOpaqueTypeError; @@ -420,7 +420,7 @@ fn borrowck_check_region_constraints<'tcx>( polonius_output, opt_closure_req, nll_errors, - polonius_diagnostics, + polonius_context, } = nll::compute_regions( root_cx, &infcx, @@ -444,7 +444,7 @@ fn borrowck_check_region_constraints<'tcx>( ®ioncx, &opt_closure_req, &borrow_set, - polonius_diagnostics.as_ref(), + polonius_context.as_ref(), ); // We also have a `#[rustc_regions]` annotation that causes us to dump @@ -486,7 +486,7 @@ fn borrowck_check_region_constraints<'tcx>( polonius_output: None, move_errors: Vec::new(), diags_buffer, - polonius_diagnostics: polonius_diagnostics.as_ref(), + polonius_context: polonius_context.as_ref(), }; struct MoveVisitor<'a, 'b, 'infcx, 'tcx> { ctxt: &'a mut MirBorrowckCtxt<'b, 'infcx, 'tcx>, @@ -525,7 +525,7 @@ fn borrowck_check_region_constraints<'tcx>( move_errors: Vec::new(), diags_buffer, polonius_output: polonius_output.as_deref(), - polonius_diagnostics: polonius_diagnostics.as_ref(), + polonius_context: polonius_context.as_ref(), }; // Compute and report region errors, if any. @@ -775,7 +775,7 @@ struct MirBorrowckCtxt<'a, 'infcx, 'tcx> { /// Results of Polonius analysis. polonius_output: Option<&'a PoloniusOutput>, /// When using `-Zpolonius=next`: the data used to compute errors and diagnostics. - polonius_diagnostics: Option<&'a PoloniusDiagnosticsContext>, + polonius_context: Option<&'a PoloniusContext>, } // Check that: diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs index 5537d90e297aa..386db0c05bc9c 100644 --- a/compiler/rustc_borrowck/src/nll.rs +++ b/compiler/rustc_borrowck/src/nll.rs @@ -22,10 +22,10 @@ use crate::borrow_set::BorrowSet; use crate::consumers::RustcFacts; use crate::diagnostics::RegionErrors; use crate::handle_placeholders::compute_sccs_applying_placeholder_outlives_constraints; +use crate::polonius::PoloniusContext; use crate::polonius::legacy::{ PoloniusFacts, PoloniusFactsExt, PoloniusLocationTable, PoloniusOutput, }; -use crate::polonius::{PoloniusContext, PoloniusDiagnosticsContext}; use crate::region_infer::RegionInferenceContext; use crate::type_check::MirTypeckRegionConstraints; use crate::type_check::free_region_relations::UniversalRegionRelations; @@ -46,7 +46,7 @@ pub(crate) struct NllOutput<'tcx> { /// When using `-Zpolonius=next`: the data used to compute errors and diagnostics, e.g. /// localized typeck and liveness constraints. - pub polonius_diagnostics: Option, + pub polonius_context: Option, } /// Rewrites the regions in the MIR to use NLL variables, also scraping out the set of universal @@ -121,7 +121,7 @@ pub(crate) fn compute_regions<'tcx>( universal_region_relations: Frozen>, constraints: MirTypeckRegionConstraints<'tcx>, mut polonius_facts: Option>, - polonius_context: Option, + mut polonius_context: Option, ) -> NllOutput<'tcx> { let polonius_output = root_cx.consumer.as_ref().map_or(false, |c| c.polonius_output()) || infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled(); @@ -153,9 +153,9 @@ pub(crate) fn compute_regions<'tcx>( // If requested for `-Zpolonius=next`, convert NLL constraints to localized outlives constraints // and use them to compute loan liveness. - let polonius_diagnostics = polonius_context.map(|polonius_context| { - polonius_context.compute_loan_liveness(infcx.tcx, &mut regioncx, body, borrow_set) - }); + if let Some(polonius_context) = polonius_context.as_mut() { + polonius_context.compute_loan_liveness(&mut regioncx, body, borrow_set) + } // If requested: dump NLL facts, and run legacy polonius analysis. let polonius_output = polonius_facts.as_ref().and_then(|polonius_facts| { @@ -188,7 +188,7 @@ pub(crate) fn compute_regions<'tcx>( polonius_output, opt_closure_req: closure_region_requirements, nll_errors, - polonius_diagnostics, + polonius_context, } } diff --git a/compiler/rustc_borrowck/src/polonius/constraints.rs b/compiler/rustc_borrowck/src/polonius/constraints.rs index 5259575785955..41a90c3d6eb4a 100644 --- a/compiler/rustc_borrowck/src/polonius/constraints.rs +++ b/compiler/rustc_borrowck/src/polonius/constraints.rs @@ -1,6 +1,19 @@ +use std::collections::BTreeMap; + +use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; +use rustc_index::interval::SparseIntervalMatrix; +use rustc_middle::mir::{Body, Location}; use rustc_middle::ty::RegionVid; use rustc_mir_dataflow::points::PointIndex; +use crate::BorrowSet; +use crate::constraints::OutlivesConstraint; +use crate::dataflow::BorrowIndex; +use crate::polonius::ConstraintDirection; +use crate::region_infer::values::LivenessValues; +use crate::type_check::Locations; +use crate::universal_regions::UniversalRegions; + /// A localized outlives constraint reifies the CFG location where the outlives constraint holds, /// within the origins themselves as if they were different from point to point: from `a: b` /// outlives constraints to `a@p: b@p`, where `p` is the point in the CFG. @@ -12,32 +25,308 @@ use rustc_mir_dataflow::points::PointIndex; /// of `q`. These depend on the liveness of the regions at these points, as well as their /// variance. /// -/// The `source` origin at `from` flows into the `target` origin at `to`. -/// /// This dual of NLL's [crate::constraints::OutlivesConstraint] therefore encodes the /// position-dependent outlives constraints used by Polonius, to model the flow-sensitive loan /// propagation via reachability within a graph of localized constraints. -#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)] -pub(crate) struct LocalizedOutlivesConstraint { - pub source: RegionVid, - pub from: PointIndex, - pub target: RegionVid, - pub to: PointIndex, +/// +/// That `LocalizedConstraintGraph` can create these edges on-demand during traversal, and we +/// therefore model them as a pair of `LocalizedNode` vertices. +/// +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub(super) struct LocalizedNode { + pub region: RegionVid, + pub point: PointIndex, +} + +/// The localized constraint graph indexes the physical and logical edges to lazily compute a given +/// node's successors during traversal. +pub(super) struct LocalizedConstraintGraph { + /// The actual, physical, edges we have recorded for a given node. We localize them on-demand + /// when traversing from the node to the successor region. + edges: FxHashMap>, + + /// The logical edges representing the outlives constraints that hold at all points in the CFG, + /// which we don't localize to avoid creating a lot of unnecessary edges in the graph. Some CFGs + /// can be big, and we don't need to create such a physical edge for every point in the CFG. + logical_edges: FxHashMap>, +} + +/// The visitor interface when traversing a `LocalizedConstraintGraph`. +pub(super) trait LocalizedConstraintGraphVisitor { + /// Callback called when traversing a given `loan` encounters a localized `node` it hasn't + /// visited before. + fn on_node_traversed(&mut self, _loan: BorrowIndex, _node: LocalizedNode) {} + + /// Callback called when discovering a new `successor` node for the `current_node`. + fn on_successor_discovered(&mut self, _current_node: LocalizedNode, _successor: LocalizedNode) { + } +} + +impl LocalizedConstraintGraph { + /// Traverses the constraints and returns the indexed graph of edges per node. + pub(super) fn new<'tcx>( + liveness: &LivenessValues, + outlives_constraints: impl Iterator>, + ) -> Self { + let mut edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default(); + let mut logical_edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default(); + + for outlives_constraint in outlives_constraints { + match outlives_constraint.locations { + Locations::All(_) => { + logical_edges + .entry(outlives_constraint.sup) + .or_default() + .insert(outlives_constraint.sub); + } + + Locations::Single(location) => { + let node = LocalizedNode { + region: outlives_constraint.sup, + point: liveness.point_from_location(location), + }; + edges.entry(node).or_default().insert(outlives_constraint.sub); + } + } + } + + LocalizedConstraintGraph { edges, logical_edges } + } + + /// Traverses the localized constraint graph per-loan, and notifies the `visitor` of discovered + /// nodes and successors. + pub(super) fn traverse<'tcx>( + &self, + body: &Body<'tcx>, + liveness: &LivenessValues, + live_region_variances: &BTreeMap, + universal_regions: &UniversalRegions<'tcx>, + borrow_set: &BorrowSet<'tcx>, + visitor: &mut impl LocalizedConstraintGraphVisitor, + ) { + let live_regions = liveness.points(); + + let mut visited = FxHashSet::default(); + let mut stack = Vec::new(); + + // Compute reachability per loan by traversing each loan's subgraph starting from where it + // is introduced. + for (loan_idx, loan) in borrow_set.iter_enumerated() { + visited.clear(); + stack.clear(); + + let start_node = LocalizedNode { + region: loan.region, + point: liveness.point_from_location(loan.reserve_location), + }; + stack.push(start_node); + + while let Some(node) = stack.pop() { + if !visited.insert(node) { + continue; + } + + // We've reached a node we haven't visited before. + let location = liveness.location_from_point(node.point); + visitor.on_node_traversed(loan_idx, node); + + // Then, we propagate the loan along the localized constraint graph. The outgoing + // edges are computed lazily, from: + // - the various physical edges present at this node, + // - the materialized logical edges that exist virtually at all points for this + // node's region, localized at this point. + + // Universal regions propagate loans along the CFG, i.e. forwards only. + let is_universal_region = universal_regions.is_universal_region(node.region); + + // The physical edges present at this node are: + // + // 1. the typeck edges that flow from region to region *at this point*. + for &succ in self.edges.get(&node).into_iter().flatten() { + let succ = LocalizedNode { region: succ, point: node.point }; + if !visited.contains(&succ) { + stack.push(succ); + visitor.on_successor_discovered(node, succ); + } + } + + // 2a. the liveness edges that flow *forward*, from this node's point to its + // successors in the CFG. + if body[location.block].statements.get(location.statement_index).is_some() { + // Intra-block edges, straight line constraints from each point to its successor + // within the same block. + let next_point = node.point + 1; + if let Some(succ) = compute_forward_successor( + node.region, + next_point, + live_regions, + live_region_variances, + is_universal_region, + ) { + if !visited.contains(&succ) { + stack.push(succ); + visitor.on_successor_discovered(node, succ); + } + } + } else { + // Inter-block edges, from the block's terminator to each successor block's + // entry point. + for successor_block in body[location.block].terminator().successors() { + let next_location = Location { block: successor_block, statement_index: 0 }; + let next_point = liveness.point_from_location(next_location); + if let Some(succ) = compute_forward_successor( + node.region, + next_point, + live_regions, + live_region_variances, + is_universal_region, + ) { + if !visited.contains(&succ) { + stack.push(succ); + visitor.on_successor_discovered(node, succ); + } + } + } + } + + // 2b. the liveness edges that flow *backward*, from this node's point to its + // predecessors in the CFG. + if !is_universal_region { + if location.statement_index > 0 { + // Backward edges to the predecessor point in the same block. + let previous_point = PointIndex::from(node.point.as_usize() - 1); + if let Some(succ) = compute_backward_successor( + node.region, + node.point, + previous_point, + live_regions, + live_region_variances, + ) { + if !visited.contains(&succ) { + stack.push(succ); + visitor.on_successor_discovered(node, succ); + } + } + } else { + // Backward edges from the block entry point to the terminator of the + // predecessor blocks. + let predecessors = body.basic_blocks.predecessors(); + for &pred_block in &predecessors[location.block] { + let previous_location = Location { + block: pred_block, + statement_index: body[pred_block].statements.len(), + }; + let previous_point = liveness.point_from_location(previous_location); + if let Some(succ) = compute_backward_successor( + node.region, + node.point, + previous_point, + live_regions, + live_region_variances, + ) { + if !visited.contains(&succ) { + stack.push(succ); + visitor.on_successor_discovered(node, succ); + } + } + } + } + } + + // And finally, we have the logical edges, materialized at this point. + for &logical_succ in self.logical_edges.get(&node.region).into_iter().flatten() { + let succ = LocalizedNode { region: logical_succ, point: node.point }; + if !visited.contains(&succ) { + stack.push(succ); + visitor.on_successor_discovered(node, succ); + } + } + } + } + } } -/// A container of [LocalizedOutlivesConstraint]s that can be turned into a traversable -/// `rustc_data_structures` graph. -#[derive(Clone, Default, Debug)] -pub(crate) struct LocalizedOutlivesConstraintSet { - pub outlives: Vec, +/// Returns the successor for the current region/point node when propagating a loan through forward +/// edges, if applicable, according to liveness and variance. +fn compute_forward_successor( + region: RegionVid, + next_point: PointIndex, + live_regions: &SparseIntervalMatrix, + live_region_variances: &BTreeMap, + is_universal_region: bool, +) -> Option { + // 1. Universal regions are semantically live at all points. + if is_universal_region { + let succ = LocalizedNode { region, point: next_point }; + return Some(succ); + } + + // 2. Otherwise, gather the edges due to explicit region liveness, when applicable. + if !live_regions.contains(region, next_point) { + return None; + } + + // Here, `region` could be live at the current point, and is live at the next point: add a + // constraint between them, according to variance. + + // Note: there currently are cases related to promoted and const generics, where we don't yet + // have variance information (possibly about temporary regions created when typeck sanitizes the + // promoteds). Until that is done, we conservatively fallback to maximizing reachability by + // adding a bidirectional edge here. This will not limit traversal whatsoever, and thus + // propagate liveness when needed. + // + // FIXME: add the missing variance information and remove this fallback bidirectional edge. + let direction = + live_region_variances.get(®ion).unwrap_or(&ConstraintDirection::Bidirectional); + + match direction { + ConstraintDirection::Backward => { + // Contravariant cases: loans flow in the inverse direction, but we're only interested + // in forward successors and there are none here. + None + } + ConstraintDirection::Forward | ConstraintDirection::Bidirectional => { + // 1. For covariant cases: loans flow in the regular direction, from the current point + // to the next point. + // 2. For invariant cases, loans can flow in both directions, but here as well, we only + // want the forward path of the bidirectional edge. + Some(LocalizedNode { region, point: next_point }) + } + } } -impl LocalizedOutlivesConstraintSet { - pub(crate) fn push(&mut self, constraint: LocalizedOutlivesConstraint) { - if constraint.source == constraint.target && constraint.from == constraint.to { - // 'a@p: 'a@p is pretty uninteresting - return; +/// Returns the successor for the current region/point node when propagating a loan through backward +/// edges, if applicable, according to liveness and variance. +fn compute_backward_successor( + region: RegionVid, + current_point: PointIndex, + previous_point: PointIndex, + live_regions: &SparseIntervalMatrix, + live_region_variances: &BTreeMap, +) -> Option { + // Liveness flows into the regions live at the next point. So, in a backwards view, we'll link + // the region from the current point, if it's live there, to the previous point. + if !live_regions.contains(region, current_point) { + return None; + } + + // FIXME: add the missing variance information and remove this fallback bidirectional edge. See + // the same comment in `compute_forward_successor`. + let direction = + live_region_variances.get(®ion).unwrap_or(&ConstraintDirection::Bidirectional); + + match direction { + ConstraintDirection::Forward => { + // Covariant cases: loans flow in the regular direction, but we're only interested in + // backward successors and there are none here. + None + } + ConstraintDirection::Backward | ConstraintDirection::Bidirectional => { + // 1. For contravariant cases: loans flow in the inverse direction, from the current + // point to the previous point. + // 2. For invariant cases, loans can flow in both directions, but here as well, we only + // want the backward path of the bidirectional edge. + Some(LocalizedNode { region, point: previous_point }) } - self.outlives.push(constraint); } } diff --git a/compiler/rustc_borrowck/src/polonius/dump.rs b/compiler/rustc_borrowck/src/polonius/dump.rs index 62f9ae173474d..606531f719f88 100644 --- a/compiler/rustc_borrowck/src/polonius/dump.rs +++ b/compiler/rustc_borrowck/src/polonius/dump.rs @@ -10,9 +10,7 @@ use rustc_session::config::MirIncludeSpans; use crate::borrow_set::BorrowSet; use crate::constraints::OutlivesConstraint; -use crate::polonius::{ - LocalizedOutlivesConstraint, LocalizedOutlivesConstraintSet, PoloniusDiagnosticsContext, -}; +use crate::polonius::{LocalizedConstraintGraphVisitor, LocalizedNode, PoloniusContext}; use crate::region_infer::values::LivenessValues; use crate::type_check::Locations; use crate::{BorrowckInferCtxt, ClosureRegionRequirements, RegionInferenceContext}; @@ -24,7 +22,7 @@ pub(crate) fn dump_polonius_mir<'tcx>( regioncx: &RegionInferenceContext<'tcx>, closure_region_requirements: &Option>, borrow_set: &BorrowSet<'tcx>, - polonius_diagnostics: Option<&PoloniusDiagnosticsContext>, + polonius_context: Option<&PoloniusContext>, ) { let tcx = infcx.tcx; if !tcx.sess.opts.unstable_opts.polonius.is_next_enabled() { @@ -33,8 +31,22 @@ pub(crate) fn dump_polonius_mir<'tcx>( let Some(dumper) = MirDumper::new(tcx, "polonius", body) else { return }; - let polonius_diagnostics = - polonius_diagnostics.expect("missing diagnostics context with `-Zpolonius=next`"); + let polonius_context = + polonius_context.expect("missing polonius context with `-Zpolonius=next`"); + + // If we have a polonius graph to dump along the rest of the MIR and NLL info, we extract its + // constraints here. + let mut collector = LocalizedOutlivesConstraintCollector { constraints: Vec::new() }; + if let Some(graph) = &polonius_context.graph { + graph.traverse( + body, + regioncx.liveness_constraints(), + &polonius_context.live_region_variances, + regioncx.universal_regions(), + borrow_set, + &mut collector, + ); + } let extra_data = &|pass_where, out: &mut dyn io::Write| { emit_polonius_mir( @@ -42,7 +54,7 @@ pub(crate) fn dump_polonius_mir<'tcx>( regioncx, closure_region_requirements, borrow_set, - &polonius_diagnostics.localized_outlives_constraints, + &collector.constraints, pass_where, out, ) @@ -60,17 +72,34 @@ pub(crate) fn dump_polonius_mir<'tcx>( let _: io::Result<()> = try { let mut file = dumper.create_dump_file("html", body)?; - emit_polonius_dump( - &dumper, - body, - regioncx, - borrow_set, - &polonius_diagnostics.localized_outlives_constraints, - &mut file, - )?; + emit_polonius_dump(&dumper, body, regioncx, borrow_set, &collector.constraints, &mut file)?; }; } +/// The constraints we'll dump as text or a mermaid graph. +struct LocalizedOutlivesConstraint { + source: RegionVid, + from: PointIndex, + target: RegionVid, + to: PointIndex, +} + +/// Visitor to record constraints encountered when traversing the localized constraint graph. +struct LocalizedOutlivesConstraintCollector { + constraints: Vec, +} + +impl LocalizedConstraintGraphVisitor for LocalizedOutlivesConstraintCollector { + fn on_successor_discovered(&mut self, current_node: LocalizedNode, successor: LocalizedNode) { + self.constraints.push(LocalizedOutlivesConstraint { + source: current_node.region, + from: current_node.point, + target: successor.region, + to: successor.point, + }); + } +} + /// The polonius dump consists of: /// - the NLL MIR /// - the list of polonius localized constraints @@ -82,7 +111,7 @@ fn emit_polonius_dump<'tcx>( body: &Body<'tcx>, regioncx: &RegionInferenceContext<'tcx>, borrow_set: &BorrowSet<'tcx>, - localized_outlives_constraints: &LocalizedOutlivesConstraintSet, + localized_outlives_constraints: &[LocalizedOutlivesConstraint], out: &mut dyn io::Write, ) -> io::Result<()> { // Prepare the HTML dump file prologue. @@ -193,7 +222,7 @@ fn emit_polonius_mir<'tcx>( regioncx: &RegionInferenceContext<'tcx>, closure_region_requirements: &Option>, borrow_set: &BorrowSet<'tcx>, - localized_outlives_constraints: &LocalizedOutlivesConstraintSet, + localized_outlives_constraints: &[LocalizedOutlivesConstraint], pass_where: PassWhere, out: &mut dyn io::Write, ) -> io::Result<()> { @@ -212,10 +241,10 @@ fn emit_polonius_mir<'tcx>( // Add localized outlives constraints match pass_where { PassWhere::BeforeCFG => { - if localized_outlives_constraints.outlives.len() > 0 { + if localized_outlives_constraints.len() > 0 { writeln!(out, "| Localized constraints")?; - for constraint in &localized_outlives_constraints.outlives { + for constraint in localized_outlives_constraints { let LocalizedOutlivesConstraint { source, from, target, to } = constraint; let from = liveness.location_from_point(*from); let to = liveness.location_from_point(*to); @@ -399,7 +428,7 @@ fn emit_mermaid_nll_sccs<'tcx>( fn emit_mermaid_constraint_graph<'tcx>( borrow_set: &BorrowSet<'tcx>, liveness: &LivenessValues, - localized_outlives_constraints: &LocalizedOutlivesConstraintSet, + localized_outlives_constraints: &[LocalizedOutlivesConstraint], out: &mut dyn io::Write, ) -> io::Result { let location_name = |location: Location| { @@ -438,7 +467,7 @@ fn emit_mermaid_constraint_graph<'tcx>( // The regions subgraphs containing the region/point nodes. let mut points_per_region: FxIndexMap> = FxIndexMap::default(); - for constraint in &localized_outlives_constraints.outlives { + for constraint in localized_outlives_constraints { points_per_region.entry(constraint.source).or_default().insert(constraint.from); points_per_region.entry(constraint.target).or_default().insert(constraint.to); } @@ -451,7 +480,7 @@ fn emit_mermaid_constraint_graph<'tcx>( } // The constraint graph edges. - for constraint in &localized_outlives_constraints.outlives { + for constraint in localized_outlives_constraints { // FIXME: add killed loans and constraint kind as edge labels. writeln!( out, @@ -463,6 +492,6 @@ fn emit_mermaid_constraint_graph<'tcx>( // Return the number of edges: this is the biggest graph in the dump and its edge count will be // mermaid's max edge count to support. - let edge_count = borrow_set.len() + localized_outlives_constraints.outlives.len(); + let edge_count = borrow_set.len() + localized_outlives_constraints.len(); Ok(edge_count) } diff --git a/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs b/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs index f1338b3bf1ee5..b6f8b4a79f39b 100644 --- a/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs +++ b/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs @@ -1,22 +1,15 @@ use std::collections::BTreeMap; use rustc_hir::def_id::DefId; -use rustc_index::bit_set::SparseBitMatrix; -use rustc_middle::mir::{Body, Location}; use rustc_middle::ty::relate::{ self, Relate, RelateResult, TypeRelation, relate_args_with_variances, }; use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt, TypeVisitable}; -use rustc_mir_dataflow::points::PointIndex; -use super::{ - ConstraintDirection, LocalizedOutlivesConstraint, LocalizedOutlivesConstraintSet, - PoloniusLivenessContext, -}; -use crate::region_infer::values::LivenessValues; +use super::{ConstraintDirection, PoloniusContext}; use crate::universal_regions::UniversalRegions; -impl PoloniusLivenessContext { +impl PoloniusContext { /// Record the variance of each region contained within the given value. pub(crate) fn record_live_region_variance<'tcx>( &mut self, @@ -34,165 +27,6 @@ impl PoloniusLivenessContext { } } -/// Propagate loans throughout the CFG: for each statement in the MIR, create localized outlives -/// constraints for loans that are propagated to the next statements. -pub(super) fn create_liveness_constraints<'tcx>( - body: &Body<'tcx>, - liveness: &LivenessValues, - live_regions: &SparseBitMatrix, - live_region_variances: &BTreeMap, - universal_regions: &UniversalRegions<'tcx>, - localized_outlives_constraints: &mut LocalizedOutlivesConstraintSet, -) { - for (block, bb) in body.basic_blocks.iter_enumerated() { - let statement_count = bb.statements.len(); - for statement_index in 0..=statement_count { - let current_location = Location { block, statement_index }; - let current_point = liveness.point_from_location(current_location); - - if statement_index < statement_count { - // Intra-block edges, straight line constraints from each point to its successor - // within the same block. - let next_location = Location { block, statement_index: statement_index + 1 }; - let next_point = liveness.point_from_location(next_location); - propagate_loans_between_points( - current_point, - next_point, - live_regions, - live_region_variances, - universal_regions, - localized_outlives_constraints, - ); - } else { - // Inter-block edges, from the block's terminator to each successor block's entry - // point. - for successor_block in bb.terminator().successors() { - let next_location = Location { block: successor_block, statement_index: 0 }; - let next_point = liveness.point_from_location(next_location); - propagate_loans_between_points( - current_point, - next_point, - live_regions, - live_region_variances, - universal_regions, - localized_outlives_constraints, - ); - } - } - } - } -} - -/// Propagate loans within a region between two points in the CFG, if that region is live at both -/// the source and target points. -fn propagate_loans_between_points( - current_point: PointIndex, - next_point: PointIndex, - live_regions: &SparseBitMatrix, - live_region_variances: &BTreeMap, - universal_regions: &UniversalRegions<'_>, - localized_outlives_constraints: &mut LocalizedOutlivesConstraintSet, -) { - // Universal regions are semantically live at all points. - // Note: we always have universal regions but they're not always (or often) involved in the - // subset graph. For now, we emit all their edges unconditionally, but some of these subgraphs - // will be disconnected from the rest of the graph and thus, unnecessary. - // - // FIXME: only emit the edges of universal regions that existential regions can reach. - for region in universal_regions.universal_regions_iter() { - localized_outlives_constraints.push(LocalizedOutlivesConstraint { - source: region, - from: current_point, - target: region, - to: next_point, - }); - } - - let Some(next_live_regions) = live_regions.row(next_point) else { - // There are no constraints to add: there are no live regions at the next point. - return; - }; - - for region in next_live_regions.iter() { - // `region` could be live at the current point, and is live at the next point: add a - // constraint between them, according to variance. - if let Some(&direction) = live_region_variances.get(®ion) { - add_liveness_constraint( - region, - current_point, - next_point, - direction, - localized_outlives_constraints, - ); - } else { - // Note: there currently are cases related to promoted and const generics, where we - // don't yet have variance information (possibly about temporary regions created when - // typeck sanitizes the promoteds). Until that is done, we conservatively fallback to - // maximizing reachability by adding a bidirectional edge here. This will not limit - // traversal whatsoever, and thus propagate liveness when needed. - // - // FIXME: add the missing variance information and remove this fallback bidirectional - // edge. - let fallback = ConstraintDirection::Bidirectional; - add_liveness_constraint( - region, - current_point, - next_point, - fallback, - localized_outlives_constraints, - ); - } - } -} - -/// Adds `LocalizedOutlivesConstraint`s between two connected points, according to the given edge -/// direction. -fn add_liveness_constraint( - region: RegionVid, - current_point: PointIndex, - next_point: PointIndex, - direction: ConstraintDirection, - localized_outlives_constraints: &mut LocalizedOutlivesConstraintSet, -) { - match direction { - ConstraintDirection::Forward => { - // Covariant cases: loans flow in the regular direction, from the current point to the - // next point. - localized_outlives_constraints.push(LocalizedOutlivesConstraint { - source: region, - from: current_point, - target: region, - to: next_point, - }); - } - ConstraintDirection::Backward => { - // Contravariant cases: loans flow in the inverse direction, from the next point to the - // current point. - localized_outlives_constraints.push(LocalizedOutlivesConstraint { - source: region, - from: next_point, - target: region, - to: current_point, - }); - } - ConstraintDirection::Bidirectional => { - // For invariant cases, loans can flow in both directions: we add both edges. - localized_outlives_constraints.push(LocalizedOutlivesConstraint { - source: region, - from: current_point, - target: region, - to: next_point, - }); - localized_outlives_constraints.push(LocalizedOutlivesConstraint { - source: region, - from: next_point, - target: region, - to: current_point, - }); - } - } -} - /// Extracts variances for regions contained within types. Follows the same structure as /// `rustc_infer`'s `Generalizer`: we try to relate a type with itself to track and extract the /// variances of regions. diff --git a/compiler/rustc_borrowck/src/polonius/loan_liveness.rs b/compiler/rustc_borrowck/src/polonius/loan_liveness.rs deleted file mode 100644 index bdc3047e5ba01..0000000000000 --- a/compiler/rustc_borrowck/src/polonius/loan_liveness.rs +++ /dev/null @@ -1,160 +0,0 @@ -use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; -use rustc_middle::ty::RegionVid; -use rustc_mir_dataflow::points::PointIndex; - -use super::{LiveLoans, LocalizedOutlivesConstraintSet}; -use crate::BorrowSet; -use crate::constraints::OutlivesConstraint; -use crate::region_infer::values::LivenessValues; -use crate::type_check::Locations; - -/// Compute loan reachability to approximately trace loan liveness throughout the CFG, by -/// traversing the full graph of constraints that combines: -/// - the localized constraints (the physical edges), -/// - with the constraints that hold at all points (the logical edges). -pub(super) fn compute_loan_liveness<'tcx>( - liveness: &LivenessValues, - outlives_constraints: impl Iterator>, - borrow_set: &BorrowSet<'tcx>, - localized_outlives_constraints: &LocalizedOutlivesConstraintSet, -) -> LiveLoans { - let mut live_loans = LiveLoans::new(borrow_set.len()); - - // Create the full graph with the physical edges we've localized earlier, and the logical edges - // of constraints that hold at all points. - let logical_constraints = - outlives_constraints.filter(|c| matches!(c.locations, Locations::All(_))); - let graph = LocalizedConstraintGraph::new(&localized_outlives_constraints, logical_constraints); - let mut visited = FxHashSet::default(); - let mut stack = Vec::new(); - - // Compute reachability per loan by traversing each loan's subgraph starting from where it is - // introduced. - for (loan_idx, loan) in borrow_set.iter_enumerated() { - visited.clear(); - stack.clear(); - - let start_node = LocalizedNode { - region: loan.region, - point: liveness.point_from_location(loan.reserve_location), - }; - stack.push(start_node); - - while let Some(node) = stack.pop() { - if !visited.insert(node) { - continue; - } - - // Record the loan as being live on entry to this point if it reaches a live region - // there. - // - // This is an approximation of liveness (which is the thing we want), in that we're - // using a single notion of reachability to represent what used to be _two_ different - // transitive closures. It didn't seem impactful when coming up with the single-graph - // and reachability through space (regions) + time (CFG) concepts, but in practice the - // combination of time-traveling with kills is more impactful than initially - // anticipated. - // - // Kills should prevent a loan from reaching its successor points in the CFG, but not - // while time-traveling: we're not actually at that CFG point, but looking for - // predecessor regions that contain the loan. One of the two TCs we had pushed the - // transitive subset edges to each point instead of having backward edges, and the - // problem didn't exist before. In the abstract, naive reachability is not enough to - // model this, we'd need a slightly different solution. For example, maybe with a - // two-step traversal: - // - at each point we first traverse the subgraph (and possibly time-travel) looking for - // exit nodes while ignoring kills, - // - and then when we're back at the current point, we continue normally. - // - // Another (less annoying) subtlety is that kills and the loan use-map are - // flow-insensitive. Kills can actually appear in places before a loan is introduced, or - // at a location that is actually unreachable in the CFG from the introduction point, - // and these can also be encountered during time-traveling. - // - // The simplest change that made sense to "fix" the issues above is taking into - // account kills that are: - // - reachable from the introduction point - // - encountered during forward traversal. Note that this is not transitive like the - // two-step traversal described above: only kills encountered on exit via a backward - // edge are ignored. - // - // This version of the analysis, however, is enough in practice to pass the tests that - // we care about and NLLs reject, without regressions on crater, and is an actionable - // subset of the full analysis. It also naturally points to areas of improvement that we - // wish to explore later, namely handling kills appropriately during traversal, instead - // of continuing traversal to all the reachable nodes. - // - // FIXME: analyze potential unsoundness, possibly in concert with a borrowck - // implementation in a-mir-formality, fuzzing, or manually crafting counter-examples. - - if liveness.is_live_at(node.region, liveness.location_from_point(node.point)) { - live_loans.insert(node.point, loan_idx); - } - - for succ in graph.outgoing_edges(node) { - stack.push(succ); - } - } - } - - live_loans -} - -/// The localized constraint graph indexes the physical and logical edges to compute a given node's -/// successors during traversal. -struct LocalizedConstraintGraph { - /// The actual, physical, edges we have recorded for a given node. - edges: FxHashMap>, - - /// The logical edges representing the outlives constraints that hold at all points in the CFG, - /// which we don't localize to avoid creating a lot of unnecessary edges in the graph. Some CFGs - /// can be big, and we don't need to create such a physical edge for every point in the CFG. - logical_edges: FxHashMap>, -} - -/// A node in the graph to be traversed, one of the two vertices of a localized outlives constraint. -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] -struct LocalizedNode { - region: RegionVid, - point: PointIndex, -} - -impl LocalizedConstraintGraph { - /// Traverses the constraints and returns the indexed graph of edges per node. - fn new<'tcx>( - constraints: &LocalizedOutlivesConstraintSet, - logical_constraints: impl Iterator>, - ) -> Self { - let mut edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default(); - for constraint in &constraints.outlives { - let source = LocalizedNode { region: constraint.source, point: constraint.from }; - let target = LocalizedNode { region: constraint.target, point: constraint.to }; - edges.entry(source).or_default().insert(target); - } - - let mut logical_edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default(); - for constraint in logical_constraints { - logical_edges.entry(constraint.sup).or_default().insert(constraint.sub); - } - - LocalizedConstraintGraph { edges, logical_edges } - } - - /// Returns the outgoing edges of a given node, not its transitive closure. - fn outgoing_edges(&self, node: LocalizedNode) -> impl Iterator { - // The outgoing edges are: - // - the physical edges present at this node, - // - the materialized logical edges that exist virtually at all points for this node's - // region, localized at this point. - let physical_edges = - self.edges.get(&node).into_iter().flat_map(|targets| targets.iter().copied()); - let materialized_edges = - self.logical_edges.get(&node.region).into_iter().flat_map(move |targets| { - targets - .iter() - .copied() - .map(move |target| LocalizedNode { point: node.point, region: target }) - }); - physical_edges.chain(materialized_edges) - } -} diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs index a9092b1981e1d..0924ce25c52ea 100644 --- a/compiler/rustc_borrowck/src/polonius/mod.rs +++ b/compiler/rustc_borrowck/src/polonius/mod.rs @@ -32,47 +32,37 @@ //! - //! - //! -//! -//! Data flows like this: -//! 1) during MIR typeck, record liveness data needed later: live region variances, as well as the -//! usual NLL liveness data (just computed on more locals). That's the [PoloniusLivenessContext]. -//! 2) once that is done, variance data is transferred, and the NLL region liveness is converted to -//! the polonius shape. That's the main [PoloniusContext]. -//! 3) during region inference, that data and the NLL outlives constraints are used to create the -//! localized outlives constraints, as described above. That's the [PoloniusDiagnosticsContext]. -//! 4) transfer this back to the main borrowck procedure: it handles computing errors and -//! diagnostics, debugging and MIR dumping concerns. mod constraints; mod dump; pub(crate) mod legacy; mod liveness_constraints; -mod loan_liveness; -mod typeck_constraints; use std::collections::BTreeMap; use rustc_data_structures::fx::FxHashSet; use rustc_index::bit_set::SparseBitMatrix; -use rustc_index::interval::SparseIntervalMatrix; use rustc_middle::mir::{Body, Local}; -use rustc_middle::ty::{RegionVid, TyCtxt}; +use rustc_middle::ty::RegionVid; use rustc_mir_dataflow::points::PointIndex; -pub(crate) use self::constraints::*; +pub(self) use self::constraints::*; pub(crate) use self::dump::dump_polonius_mir; -use self::liveness_constraints::create_liveness_constraints; -use self::loan_liveness::compute_loan_liveness; -use self::typeck_constraints::convert_typeck_constraints; use crate::dataflow::BorrowIndex; +use crate::region_infer::values::LivenessValues; use crate::{BorrowSet, RegionInferenceContext}; pub(crate) type LiveLoans = SparseBitMatrix; -/// This struct holds the liveness data created during MIR typeck, and which will be used later in -/// the process, to compute the polonius localized constraints. +/// This struct holds the necessary +/// - liveness data, created during MIR typeck, and which will be used to lazily compute the +/// polonius localized constraints, during NLL region inference as well as MIR dumping, +/// - data needed by the borrowck error computation and diagnostics. #[derive(Default)] -pub(crate) struct PoloniusLivenessContext { +pub(crate) struct PoloniusContext { + /// The graph from which we extract the localized outlives constraints. + graph: Option, + /// The expected edge direction per live region: the kind of directed edge we'll create as /// liveness constraints depends on the variance of types with respect to each contained region. live_region_variances: BTreeMap, @@ -84,27 +74,6 @@ pub(crate) struct PoloniusLivenessContext { pub(crate) boring_nll_locals: FxHashSet, } -/// This struct holds the data needed to create the Polonius localized constraints. Its data is -/// transferred and converted from the [PoloniusLivenessContext] at the end of MIR typeck. -pub(crate) struct PoloniusContext { - /// The liveness data we recorded during MIR typeck. - liveness_context: PoloniusLivenessContext, - - /// The set of regions that are live at a given point in the CFG, used to create localized - /// outlives constraints between regions that are live at connected points in the CFG. - live_regions: SparseBitMatrix, -} - -/// This struct holds the data needed by the borrowck error computation and diagnostics. Its data is -/// computed from the [PoloniusContext] when computing NLL regions. -pub(crate) struct PoloniusDiagnosticsContext { - /// The localized outlives constraints that were computed in the main analysis. - localized_outlives_constraints: LocalizedOutlivesConstraintSet, - - /// The liveness data computed during MIR typeck: [PoloniusLivenessContext::boring_nll_locals]. - pub(crate) boring_nll_locals: FxHashSet, -} - /// The direction a constraint can flow into. Used to create liveness constraints according to /// variance. #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -120,26 +89,6 @@ enum ConstraintDirection { } impl PoloniusContext { - /// Unlike NLLs, in polonius we traverse the cfg to look for regions live across an edge, so we - /// need to transpose the "points where each region is live" matrix to a "live regions per point" - /// matrix. - // FIXME: avoid this conversion by always storing liveness data in this shape in the rest of - // borrowck. - pub(crate) fn create_from_liveness( - liveness_context: PoloniusLivenessContext, - num_regions: usize, - points_per_live_region: &SparseIntervalMatrix, - ) -> PoloniusContext { - let mut live_regions_per_point = SparseBitMatrix::new(num_regions); - for region in points_per_live_region.rows() { - for point in points_per_live_region.row(region).unwrap().iter() { - live_regions_per_point.insert(point, region); - } - } - - PoloniusContext { live_regions: live_regions_per_point, liveness_context } - } - /// Computes live loans using the set of loans model for `-Zpolonius=next`. /// /// First, creates a constraint graph combining regions and CFG points, by: @@ -151,44 +100,91 @@ impl PoloniusContext { /// /// The constraint data will be used to compute errors and diagnostics. pub(crate) fn compute_loan_liveness<'tcx>( - self, - tcx: TyCtxt<'tcx>, + &mut self, regioncx: &mut RegionInferenceContext<'tcx>, body: &Body<'tcx>, borrow_set: &BorrowSet<'tcx>, - ) -> PoloniusDiagnosticsContext { - let PoloniusLivenessContext { live_region_variances, boring_nll_locals } = - self.liveness_context; - - let mut localized_outlives_constraints = LocalizedOutlivesConstraintSet::default(); - convert_typeck_constraints( - tcx, - body, - regioncx.liveness_constraints(), - regioncx.outlives_constraints(), - regioncx.universal_regions(), - &mut localized_outlives_constraints, - ); - - create_liveness_constraints( - body, - regioncx.liveness_constraints(), - &self.live_regions, - &live_region_variances, - regioncx.universal_regions(), - &mut localized_outlives_constraints, - ); - - // Now that we have a complete graph, we can compute reachability to trace the liveness of - // loans for the next step in the chain, the NLL loan scope and active loans computations. - let live_loans = compute_loan_liveness( - regioncx.liveness_constraints(), - regioncx.outlives_constraints(), - borrow_set, - &localized_outlives_constraints, - ); - regioncx.record_live_loans(live_loans); - - PoloniusDiagnosticsContext { localized_outlives_constraints, boring_nll_locals } + ) { + let liveness = regioncx.liveness_constraints(); + + // We don't need to prepare the graph (index NLL constraints, etc.) if we have no loans to + // trace throughout localized constraints. + if borrow_set.len() > 0 { + // From the outlives constraints, liveness, and variances, we can compute reachability + // on the lazy localized constraint graph to trace the liveness of loans, for the next + // step in the chain (the NLL loan scope and active loans computations). + let graph = LocalizedConstraintGraph::new(liveness, regioncx.outlives_constraints()); + + let mut live_loans = LiveLoans::new(borrow_set.len()); + let mut visitor = LoanLivenessVisitor { liveness, live_loans: &mut live_loans }; + graph.traverse( + body, + liveness, + &self.live_region_variances, + regioncx.universal_regions(), + borrow_set, + &mut visitor, + ); + regioncx.record_live_loans(live_loans); + + // The graph can be traversed again during MIR dumping, so we store it here. + self.graph = Some(graph); + } + } +} + +/// Visitor to record loan liveness when traversing the localized constraint graph. +struct LoanLivenessVisitor<'a> { + liveness: &'a LivenessValues, + live_loans: &'a mut LiveLoans, +} + +impl LocalizedConstraintGraphVisitor for LoanLivenessVisitor<'_> { + fn on_node_traversed(&mut self, loan: BorrowIndex, node: LocalizedNode) { + // Record the loan as being live on entry to this point if it reaches a live region + // there. + // + // This is an approximation of liveness (which is the thing we want), in that we're + // using a single notion of reachability to represent what used to be _two_ different + // transitive closures. It didn't seem impactful when coming up with the single-graph + // and reachability through space (regions) + time (CFG) concepts, but in practice the + // combination of time-traveling with kills is more impactful than initially + // anticipated. + // + // Kills should prevent a loan from reaching its successor points in the CFG, but not + // while time-traveling: we're not actually at that CFG point, but looking for + // predecessor regions that contain the loan. One of the two TCs we had pushed the + // transitive subset edges to each point instead of having backward edges, and the + // problem didn't exist before. In the abstract, naive reachability is not enough to + // model this, we'd need a slightly different solution. For example, maybe with a + // two-step traversal: + // - at each point we first traverse the subgraph (and possibly time-travel) looking for + // exit nodes while ignoring kills, + // - and then when we're back at the current point, we continue normally. + // + // Another (less annoying) subtlety is that kills and the loan use-map are + // flow-insensitive. Kills can actually appear in places before a loan is introduced, or + // at a location that is actually unreachable in the CFG from the introduction point, + // and these can also be encountered during time-traveling. + // + // The simplest change that made sense to "fix" the issues above is taking into account + // kills that are: + // - reachable from the introduction point + // - encountered during forward traversal. Note that this is not transitive like the + // two-step traversal described above: only kills encountered on exit via a backward + // edge are ignored. + // + // This version of the analysis, however, is enough in practice to pass the tests that + // we care about and NLLs reject, without regressions on crater, and is an actionable + // subset of the full analysis. It also naturally points to areas of improvement that we + // wish to explore later, namely handling kills appropriately during traversal, instead + // of continuing traversal to all the reachable nodes. + // + // FIXME: analyze potential unsoundness, possibly in concert with a borrowck + // implementation in a-mir-formality, fuzzing, or manually crafting counter-examples. + let location = self.liveness.location_from_point(node.point); + if self.liveness.is_live_at(node.region, location) { + self.live_loans.insert(node.point, loan); + } } } diff --git a/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs b/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs deleted file mode 100644 index e4e52962bf7f9..0000000000000 --- a/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs +++ /dev/null @@ -1,218 +0,0 @@ -use rustc_data_structures::fx::FxHashSet; -use rustc_middle::mir::{Body, Location, Statement, StatementKind, Terminator, TerminatorKind}; -use rustc_middle::ty::{TyCtxt, TypeVisitable}; -use rustc_mir_dataflow::points::PointIndex; - -use super::{LocalizedOutlivesConstraint, LocalizedOutlivesConstraintSet}; -use crate::constraints::OutlivesConstraint; -use crate::region_infer::values::LivenessValues; -use crate::type_check::Locations; -use crate::universal_regions::UniversalRegions; - -/// Propagate loans throughout the subset graph at a given point (with some subtleties around the -/// location where effects start to be visible). -pub(super) fn convert_typeck_constraints<'tcx>( - tcx: TyCtxt<'tcx>, - body: &Body<'tcx>, - liveness: &LivenessValues, - outlives_constraints: impl Iterator>, - universal_regions: &UniversalRegions<'tcx>, - localized_outlives_constraints: &mut LocalizedOutlivesConstraintSet, -) { - for outlives_constraint in outlives_constraints { - match outlives_constraint.locations { - Locations::All(_) => { - // We don't turn constraints holding at all points into physical edges at every - // point in the graph. They are encoded into *traversal* instead: a given node's - // successors will combine these logical edges with the regular, physical, localized - // edges. - continue; - } - - Locations::Single(location) => { - // This constraint is marked as holding at one location, we localize it to that - // location or its successor, depending on the corresponding MIR - // statement/terminator. Unfortunately, they all show up from typeck as coming "on - // entry", so for now we modify them to take effects that should apply "on exit" - // into account. - // - // FIXME: this approach is subtle, complicated, and hard to test, so we should track - // this information better in MIR typeck instead, for example with a new `Locations` - // variant that contains which node is crossing over between entry and exit. - let point = liveness.point_from_location(location); - let localized_constraint = if let Some(stmt) = - body[location.block].statements.get(location.statement_index) - { - localize_statement_constraint( - tcx, - body, - stmt, - &outlives_constraint, - point, - universal_regions, - ) - } else { - assert_eq!(location.statement_index, body[location.block].statements.len()); - let terminator = body[location.block].terminator(); - localize_terminator_constraint( - tcx, - body, - terminator, - liveness, - &outlives_constraint, - point, - universal_regions, - ) - }; - localized_outlives_constraints.push(localized_constraint); - } - } - } -} - -/// For a given outlives constraint arising from a MIR statement, localize the constraint with the -/// needed CFG `from`-`to` intra-block nodes. -fn localize_statement_constraint<'tcx>( - tcx: TyCtxt<'tcx>, - body: &Body<'tcx>, - stmt: &Statement<'tcx>, - outlives_constraint: &OutlivesConstraint<'tcx>, - current_point: PointIndex, - universal_regions: &UniversalRegions<'tcx>, -) -> LocalizedOutlivesConstraint { - match &stmt.kind { - StatementKind::Assign(box (lhs, rhs)) => { - // To create localized outlives constraints without midpoints, we rely on the property - // that no input regions from the RHS of the assignment will flow into themselves: they - // should not appear in the output regions in the LHS. We believe this to be true by - // construction of the MIR, via temporaries, and assert it here. - // - // We think we don't need midpoints because: - // - every LHS Place has a unique set of regions that don't appear elsewhere - // - this implies that for them to be part of the RHS, the same Place must be read and - // written - // - and that should be impossible in MIR - // - // When we have a more complete implementation in the future, tested with crater, etc, - // we can remove this assertion. It's a debug assert because it can be expensive. - debug_assert!( - { - let mut lhs_regions = FxHashSet::default(); - tcx.for_each_free_region(lhs, |region| { - let region = universal_regions.to_region_vid(region); - lhs_regions.insert(region); - }); - - let mut rhs_regions = FxHashSet::default(); - tcx.for_each_free_region(rhs, |region| { - let region = universal_regions.to_region_vid(region); - rhs_regions.insert(region); - }); - - // The intersection between LHS and RHS regions should be empty. - lhs_regions.is_disjoint(&rhs_regions) - }, - "there should be no common regions between the LHS and RHS of an assignment" - ); - - let lhs_ty = body.local_decls[lhs.local].ty; - let successor_point = current_point; - compute_constraint_direction( - tcx, - outlives_constraint, - &lhs_ty, - current_point, - successor_point, - universal_regions, - ) - } - _ => { - // For the other cases, we localize an outlives constraint to where it arises. - LocalizedOutlivesConstraint { - source: outlives_constraint.sup, - from: current_point, - target: outlives_constraint.sub, - to: current_point, - } - } - } -} - -/// For a given outlives constraint arising from a MIR terminator, localize the constraint with the -/// needed CFG `from`-`to` inter-block nodes. -fn localize_terminator_constraint<'tcx>( - tcx: TyCtxt<'tcx>, - body: &Body<'tcx>, - terminator: &Terminator<'tcx>, - liveness: &LivenessValues, - outlives_constraint: &OutlivesConstraint<'tcx>, - current_point: PointIndex, - universal_regions: &UniversalRegions<'tcx>, -) -> LocalizedOutlivesConstraint { - // FIXME: check if other terminators need the same handling as `Call`s, in particular - // Assert/Yield/Drop. A handful of tests are failing with Drop related issues, as well as some - // coroutine tests, and that may be why. - match &terminator.kind { - // FIXME: also handle diverging calls. - TerminatorKind::Call { destination, target: Some(target), .. } => { - // Calls are similar to assignments, and thus follow the same pattern. If there is a - // target for the call we also relate what flows into the destination here to entry to - // that successor. - let destination_ty = destination.ty(&body.local_decls, tcx); - let successor_location = Location { block: *target, statement_index: 0 }; - let successor_point = liveness.point_from_location(successor_location); - compute_constraint_direction( - tcx, - outlives_constraint, - &destination_ty, - current_point, - successor_point, - universal_regions, - ) - } - _ => { - // Typeck constraints guide loans between regions at the current point, so we do that in - // the general case, and liveness will take care of making them flow to the terminator's - // successors. - LocalizedOutlivesConstraint { - source: outlives_constraint.sup, - from: current_point, - target: outlives_constraint.sub, - to: current_point, - } - } - } -} - -/// For a given outlives constraint and CFG edge, returns the localized constraint with the -/// appropriate `from`-`to` direction. This is computed according to whether the constraint flows to -/// or from a free region in the given `value`, some kind of result for an effectful operation, like -/// the LHS of an assignment. -fn compute_constraint_direction<'tcx>( - tcx: TyCtxt<'tcx>, - outlives_constraint: &OutlivesConstraint<'tcx>, - value: &impl TypeVisitable>, - current_point: PointIndex, - successor_point: PointIndex, - universal_regions: &UniversalRegions<'tcx>, -) -> LocalizedOutlivesConstraint { - let mut to = current_point; - let mut from = current_point; - tcx.for_each_free_region(value, |region| { - let region = universal_regions.to_region_vid(region); - if region == outlives_constraint.sub { - // This constraint flows into the result, its effects start becoming visible on exit. - to = successor_point; - } else if region == outlives_constraint.sup { - // This constraint flows from the result, its effects start becoming visible on exit. - from = successor_point; - } - }); - - LocalizedOutlivesConstraint { - source: outlives_constraint.sup, - from, - target: outlives_constraint.sub, - to, - } -} diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs index ca1b850f7665d..442c37e26ec18 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs @@ -11,7 +11,7 @@ use tracing::debug; use super::TypeChecker; use crate::constraints::OutlivesConstraintSet; -use crate::polonius::PoloniusLivenessContext; +use crate::polonius::PoloniusContext; use crate::region_infer::values::LivenessValues; use crate::universal_regions::UniversalRegions; @@ -48,7 +48,7 @@ pub(super) fn generate<'tcx>( if typeck.tcx().sess.opts.unstable_opts.polonius.is_next_enabled() { let (_, boring_locals) = compute_relevant_live_locals(typeck.tcx(), &free_regions, typeck.body); - typeck.polonius_liveness.as_mut().unwrap().boring_nll_locals = + typeck.polonius_context.as_mut().unwrap().boring_nll_locals = boring_locals.into_iter().collect(); free_regions = typeck.universal_regions.universal_regions_iter().collect(); } @@ -63,7 +63,7 @@ pub(super) fn generate<'tcx>( typeck.tcx(), &mut typeck.constraints.liveness_constraints, &typeck.universal_regions, - &mut typeck.polonius_liveness, + &mut typeck.polonius_context, typeck.body, ); } @@ -140,11 +140,11 @@ fn record_regular_live_regions<'tcx>( tcx: TyCtxt<'tcx>, liveness_constraints: &mut LivenessValues, universal_regions: &UniversalRegions<'tcx>, - polonius_liveness: &mut Option, + polonius_context: &mut Option, body: &Body<'tcx>, ) { let mut visitor = - LiveVariablesVisitor { tcx, liveness_constraints, universal_regions, polonius_liveness }; + LiveVariablesVisitor { tcx, liveness_constraints, universal_regions, polonius_context }; for (bb, data) in body.basic_blocks.iter_enumerated() { visitor.visit_basic_block_data(bb, data); } @@ -155,7 +155,7 @@ struct LiveVariablesVisitor<'a, 'tcx> { tcx: TyCtxt<'tcx>, liveness_constraints: &'a mut LivenessValues, universal_regions: &'a UniversalRegions<'tcx>, - polonius_liveness: &'a mut Option, + polonius_context: &'a mut Option, } impl<'a, 'tcx> Visitor<'tcx> for LiveVariablesVisitor<'a, 'tcx> { @@ -207,8 +207,8 @@ impl<'a, 'tcx> LiveVariablesVisitor<'a, 'tcx> { }); // When using `-Zpolonius=next`, we record the variance of each live region. - if let Some(polonius_liveness) = self.polonius_liveness { - polonius_liveness.record_live_region_variance(self.tcx, self.universal_regions, value); + if let Some(polonius_context) = self.polonius_context { + polonius_context.record_live_region_variance(self.tcx, self.universal_regions, value); } } } diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs index 7ac94020de03b..840210496eb44 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs @@ -622,8 +622,8 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> { }); // When using `-Zpolonius=next`, we record the variance of each live region. - if let Some(polonius_liveness) = typeck.polonius_liveness.as_mut() { - polonius_liveness.record_live_region_variance( + if let Some(polonius_context) = typeck.polonius_context.as_mut() { + polonius_context.record_live_region_variance( typeck.infcx.tcx, typeck.universal_regions, value, diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 097416b4f2804..17af896c4a22f 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -42,8 +42,8 @@ use tracing::{debug, instrument, trace}; use crate::borrow_set::BorrowSet; use crate::constraints::{OutlivesConstraint, OutlivesConstraintSet}; use crate::diagnostics::UniverseInfo; +use crate::polonius::PoloniusContext; use crate::polonius::legacy::{PoloniusFacts, PoloniusLocationTable}; -use crate::polonius::{PoloniusContext, PoloniusLivenessContext}; use crate::region_infer::TypeTest; use crate::region_infer::values::{LivenessValues, PlaceholderIndex, PlaceholderIndices}; use crate::session_diagnostics::{MoveUnsized, SimdIntrinsicArgConst}; @@ -136,8 +136,8 @@ pub(crate) fn type_check<'tcx>( debug!(?normalized_inputs_and_output); - let polonius_liveness = if infcx.tcx.sess.opts.unstable_opts.polonius.is_next_enabled() { - Some(PoloniusLivenessContext::default()) + let polonius_context = if infcx.tcx.sess.opts.unstable_opts.polonius.is_next_enabled() { + Some(PoloniusContext::default()) } else { None }; @@ -159,7 +159,7 @@ pub(crate) fn type_check<'tcx>( borrow_set, constraints: &mut constraints, deferred_closure_requirements: &mut deferred_closure_requirements, - polonius_liveness, + polonius_context, }; typeck.check_user_type_annotations(); @@ -169,14 +169,7 @@ pub(crate) fn type_check<'tcx>( liveness::generate(&mut typeck, &location_map, move_data); - // We're done with typeck, we can finalize the polonius liveness context for region inference. - let polonius_context = typeck.polonius_liveness.take().map(|liveness_context| { - PoloniusContext::create_from_liveness( - liveness_context, - infcx.num_region_vars(), - typeck.constraints.liveness_constraints.points(), - ) - }); + let polonius_context = typeck.polonius_context; // In case type check encountered an error region, we suppress unhelpful extra // errors in by clearing out all outlives bounds that we may end up checking. @@ -235,7 +228,7 @@ struct TypeChecker<'a, 'tcx> { constraints: &'a mut MirTypeckRegionConstraints<'tcx>, deferred_closure_requirements: &'a mut DeferredClosureRequirements<'tcx>, /// When using `-Zpolonius=next`, the liveness helper data used to create polonius constraints. - polonius_liveness: Option, + polonius_context: Option, } /// Holder struct for passing results from MIR typeck to the rest of the non-lexical regions