Use SparseIntervalMatrix instead of SparseBitMatrix

Region inference contains several bitsets which are filled with large intervals
representing liveness. These can cause excessive memory usage, and are
relatively slow when growing to large sizes compared to the IntervalSet.

Author: Mark-Simulacrum

compiler/rustc_borrowck/src/region_infer/values.rs

@@ -1,5 +1,7 @@
 use rustc_data_structures::fx::FxIndexSet;
-use rustc_index::bit_set::{HybridBitSet, SparseBitMatrix};
+use rustc_index::bit_set::SparseBitMatrix;
+use rustc_index::interval::IntervalSet;
+use rustc_index::interval::SparseIntervalMatrix;
 use rustc_index::vec::Idx;
 use rustc_index::vec::IndexVec;
 use rustc_middle::mir::{BasicBlock, Body, Location};
@@ -110,19 +112,19 @@ crate enum RegionElement {
     PlaceholderRegion(ty::PlaceholderRegion),
 }
 
-/// When we initially compute liveness, we use a bit matrix storing
-/// points for each region-vid.
+/// When we initially compute liveness, we use an interval matrix storing
+/// liveness ranges for each region-vid.
 crate struct LivenessValues<N: Idx> {
     elements: Rc<RegionValueElements>,
-    points: SparseBitMatrix<N, PointIndex>,
+    points: SparseIntervalMatrix<N, PointIndex>,
 }
 
 impl<N: Idx> LivenessValues<N> {
     /// Creates a new set of "region values" that tracks causal information.
     /// Each of the regions in num_region_variables will be initialized with an
     /// empty set of points and no causal information.
     crate fn new(elements: Rc<RegionValueElements>) -> Self {
-        Self { points: SparseBitMatrix::new(elements.num_points), elements }
+        Self { points: SparseIntervalMatrix::new(elements.num_points), elements }
     }
 
     /// Iterate through each region that has a value in this set.
@@ -140,7 +142,7 @@ impl<N: Idx> LivenessValues<N> {
 
     /// Adds all the elements in the given bit array into the given
     /// region. Returns whether any of them are newly added.
-    crate fn add_elements(&mut self, row: N, locations: &HybridBitSet<PointIndex>) -> bool {
+    crate fn add_elements(&mut self, row: N, locations: &IntervalSet<PointIndex>) -> bool {
         debug!("LivenessValues::add_elements(row={:?}, locations={:?})", row, locations);
         self.points.union_row(row, locations)
     }
@@ -153,7 +155,7 @@ impl<N: Idx> LivenessValues<N> {
     /// Returns `true` if the region `r` contains the given element.
     crate fn contains(&self, row: N, location: Location) -> bool {
         let index = self.elements.point_from_location(location);
-        self.points.contains(row, index)
+        self.points.row(row).map_or(false, |r| r.contains(index))
     }
 
     /// Returns an iterator of all the elements contained by the region `r`
@@ -221,7 +223,7 @@ impl PlaceholderIndices {
 crate struct RegionValues<N: Idx> {
     elements: Rc<RegionValueElements>,
     placeholder_indices: Rc<PlaceholderIndices>,
-    points: SparseBitMatrix<N, PointIndex>,
+    points: SparseIntervalMatrix<N, PointIndex>,
     free_regions: SparseBitMatrix<N, RegionVid>,
 
     /// Placeholders represent bound regions -- so something like `'a`
@@ -241,7 +243,7 @@ impl<N: Idx> RegionValues<N> {
         let num_placeholders = placeholder_indices.len();
         Self {
             elements: elements.clone(),
-            points: SparseBitMatrix::new(elements.num_points),
+            points: SparseIntervalMatrix::new(elements.num_points),
             placeholder_indices: placeholder_indices.clone(),
             free_regions: SparseBitMatrix::new(num_universal_regions),
             placeholders: SparseBitMatrix::new(num_placeholders),

compiler/rustc_borrowck/src/type_check/liveness/trace.rs

@@ -1,5 +1,6 @@
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_index::bit_set::HybridBitSet;
+use rustc_index::interval::IntervalSet;
 use rustc_infer::infer::canonical::QueryRegionConstraints;
 use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, Local, Location};
 use rustc_middle::ty::{Ty, TypeFoldable};
@@ -105,12 +106,12 @@ struct LivenessResults<'me, 'typeck, 'flow, 'tcx> {
 
     /// Points where the current variable is "use live" -- meaning
     /// that there is a future "full use" that may use its value.
-    use_live_at: HybridBitSet<PointIndex>,
+    use_live_at: IntervalSet<PointIndex>,
 
     /// Points where the current variable is "drop live" -- meaning
     /// that there is no future "full use" that may use its value, but
     /// there is a future drop.
-    drop_live_at: HybridBitSet<PointIndex>,
+    drop_live_at: IntervalSet<PointIndex>,
 
     /// Locations where drops may occur.
     drop_locations: Vec<Location>,
@@ -125,8 +126,8 @@ impl<'me, 'typeck, 'flow, 'tcx> LivenessResults<'me, 'typeck, 'flow, 'tcx> {
         LivenessResults {
             cx,
             defs: HybridBitSet::new_empty(num_points),
-            use_live_at: HybridBitSet::new_empty(num_points),
-            drop_live_at: HybridBitSet::new_empty(num_points),
+            use_live_at: IntervalSet::new(num_points),
+            drop_live_at: IntervalSet::new(num_points),
             drop_locations: vec![],
             stack: vec![],
         }
@@ -165,7 +166,7 @@ impl<'me, 'typeck, 'flow, 'tcx> LivenessResults<'me, 'typeck, 'flow, 'tcx> {
         drop_used: Vec<(Local, Location)>,
         live_locals: FxHashSet<Local>,
     ) {
-        let locations = HybridBitSet::new_empty(self.cx.elements.num_points());
+        let locations = IntervalSet::new(self.cx.elements.num_points());
 
         for (local, location) in drop_used {
             if !live_locals.contains(&local) {
@@ -456,7 +457,7 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
     fn add_use_live_facts_for(
         &mut self,
         value: impl TypeFoldable<'tcx>,
-        live_at: &HybridBitSet<PointIndex>,
+        live_at: &IntervalSet<PointIndex>,
     ) {
         debug!("add_use_live_facts_for(value={:?})", value);
 
@@ -473,7 +474,7 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
         dropped_local: Local,
         dropped_ty: Ty<'tcx>,
         drop_locations: &[Location],
-        live_at: &HybridBitSet<PointIndex>,
+        live_at: &IntervalSet<PointIndex>,
     ) {
         debug!(
             "add_drop_live_constraint(\
@@ -521,7 +522,7 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
         elements: &RegionValueElements,
         typeck: &mut TypeChecker<'_, 'tcx>,
         value: impl TypeFoldable<'tcx>,
-        live_at: &HybridBitSet<PointIndex>,
+        live_at: &IntervalSet<PointIndex>,
     ) {
         debug!("make_all_regions_live(value={:?})", value);
         debug!(