|
| 1 | +//! This module handles the locking logic during compilation. |
| 2 | +//! |
| 3 | +//! The locking scheme is based on build unit level locking. |
| 4 | +//! Each build unit consists of a partial and full lock used to represent multiple lock states. |
| 5 | +//! |
| 6 | +//! | State | `partial.lock` | `full.lock` | |
| 7 | +//! |------------------------|----------------|--------------| |
| 8 | +//! | Unlocked | `unlocked` | `unlocked` | |
| 9 | +//! | Building Exclusive | `exclusive` | `exclusive` | |
| 10 | +//! | Building Non-Exclusive | `shared` | `exclusive` | |
| 11 | +//! | Shared Partial | `shared` | `unlocked` | |
| 12 | +//! | Shared Full | `shared` | `shared` | |
| 13 | +//! |
| 14 | +//! Generally a build unit will full the following flow: |
| 15 | +//! 1. Acquire a "building exclusive" lock for the current build unit. |
| 16 | +//! 2. Acquire "shared" locks on all dependency build units. |
| 17 | +//! 3. Begin building with rustc |
| 18 | +//! 4. If we are building a library, downgrade to a "building non-exclusive" lock when the `.rmeta` has been generated. |
| 19 | +//! 5. Once complete release all locks. |
| 20 | +//! |
| 21 | +//! Most build units only require metadata (.rmeta) from dependencies, so they can begin building |
| 22 | +//! once the dependency units have produced the .rmeta. These units take a "shared partial" lock |
| 23 | +//! which can be taken while the dependency still holds the "build non-exclusive" lock. |
| 24 | +//! |
| 25 | +//! Note that some build unit types like bin and proc-macros require the full dependency build |
| 26 | +//! (.rlib). For these unit types they must take a "shared full" lock on dependency units which will |
| 27 | +//! block until the dependency unit is fully built. |
| 28 | +//! |
| 29 | +//! The primary reason for the complexity here it to enable fine grain locking while also allowing pipelined builds. |
| 30 | +//! |
| 31 | +//! [`CompilationLock`] is the primary interface for locking. |
| 32 | +
|
| 33 | +use std::{ |
| 34 | + collections::HashSet, |
| 35 | + fs::{File, OpenOptions}, |
| 36 | + path::{Path, PathBuf}, |
| 37 | +}; |
| 38 | + |
| 39 | +use anyhow::Context; |
| 40 | +use itertools::Itertools; |
| 41 | +use tracing::{instrument, trace}; |
| 42 | + |
| 43 | +use crate::{ |
| 44 | + CargoResult, |
| 45 | + core::compiler::{BuildRunner, Unit, layout::BuildUnitLockLocation}, |
| 46 | +}; |
| 47 | + |
| 48 | +/// The locking mode that will be used for build-dir. |
| 49 | +#[derive(Debug)] |
| 50 | +pub enum LockingMode { |
| 51 | + /// Fine grain locking (Build unit level) |
| 52 | + Fine, |
| 53 | + /// Coarse grain locking (Profile level) |
| 54 | + Coarse, |
| 55 | +} |
| 56 | + |
| 57 | +/// The type of lock to take when taking a shared lock. |
| 58 | +/// See the module documentation for more information about shared lock types. |
| 59 | +#[derive(Debug)] |
| 60 | +pub enum SharedLockType { |
| 61 | + /// A shared lock that might still be compiling a .rlib |
| 62 | + Partial, |
| 63 | + /// A shared lock that is guaranteed to not be compiling |
| 64 | + Full, |
| 65 | +} |
| 66 | + |
| 67 | +/// A lock for compiling a build unit. |
| 68 | +/// |
| 69 | +/// Internally this lock is made up of many [`UnitLock`]s for the unit and it's dependencies. |
| 70 | +pub struct CompilationLock { |
| 71 | + /// The path to the lock file of the unit to compile |
| 72 | + unit: UnitLock, |
| 73 | + /// The paths to lock files of the unit's dependencies |
| 74 | + dependency_units: Vec<UnitLock>, |
| 75 | +} |
| 76 | + |
| 77 | +impl CompilationLock { |
| 78 | + pub fn new(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> Self { |
| 79 | + let unit_lock = UnitLock::new(build_runner.files().build_unit_lock(unit)); |
| 80 | + |
| 81 | + let dependency_units = all_dependency_units(build_runner, unit) |
| 82 | + .into_iter() |
| 83 | + .map(|unit| UnitLock::new(build_runner.files().build_unit_lock(&unit))) |
| 84 | + .collect_vec(); |
| 85 | + |
| 86 | + Self { |
| 87 | + unit: unit_lock, |
| 88 | + dependency_units, |
| 89 | + } |
| 90 | + } |
| 91 | + |
| 92 | + #[instrument(skip(self))] |
| 93 | + pub fn lock(&mut self, ty: &SharedLockType) -> CargoResult<()> { |
| 94 | + self.unit.lock_exclusive()?; |
| 95 | + |
| 96 | + for d in self.dependency_units.iter_mut() { |
| 97 | + d.lock_shared(ty)?; |
| 98 | + } |
| 99 | + |
| 100 | + trace!("acquired lock: {:?}", self.unit.partial.parent()); |
| 101 | + |
| 102 | + Ok(()) |
| 103 | + } |
| 104 | + |
| 105 | + pub fn rmeta_produced(&mut self) -> CargoResult<()> { |
| 106 | + trace!("downgrading lock: {:?}", self.unit.partial.parent()); |
| 107 | + |
| 108 | + // Downgrade the lock on the unit we are building so that we can unblock other units to |
| 109 | + // compile. We do not need to downgrade our dependency locks since they should always be a |
| 110 | + // shared lock. |
| 111 | + self.unit.downgrade()?; |
| 112 | + |
| 113 | + Ok(()) |
| 114 | + } |
| 115 | +} |
| 116 | + |
| 117 | +/// A lock for a single build unit. |
| 118 | +struct UnitLock { |
| 119 | + partial: PathBuf, |
| 120 | + full: PathBuf, |
| 121 | + guard: Option<UnitLockGuard>, |
| 122 | +} |
| 123 | + |
| 124 | +struct UnitLockGuard { |
| 125 | + partial: File, |
| 126 | + _full: Option<File>, |
| 127 | +} |
| 128 | + |
| 129 | +impl UnitLock { |
| 130 | + pub fn new(location: BuildUnitLockLocation) -> Self { |
| 131 | + Self { |
| 132 | + partial: location.partial, |
| 133 | + full: location.full, |
| 134 | + guard: None, |
| 135 | + } |
| 136 | + } |
| 137 | + |
| 138 | + pub fn lock_exclusive(&mut self) -> CargoResult<()> { |
| 139 | + assert!(self.guard.is_none()); |
| 140 | + |
| 141 | + let partial = open_file(&self.partial)?; |
| 142 | + partial.lock()?; |
| 143 | + |
| 144 | + let full = open_file(&self.full)?; |
| 145 | + full.lock()?; |
| 146 | + |
| 147 | + self.guard = Some(UnitLockGuard { |
| 148 | + partial, |
| 149 | + _full: Some(full), |
| 150 | + }); |
| 151 | + Ok(()) |
| 152 | + } |
| 153 | + |
| 154 | + pub fn lock_shared(&mut self, ty: &SharedLockType) -> CargoResult<()> { |
| 155 | + assert!(self.guard.is_none()); |
| 156 | + |
| 157 | + let partial = open_file(&self.partial)?; |
| 158 | + partial.lock_shared()?; |
| 159 | + |
| 160 | + let full = if matches!(ty, SharedLockType::Full) { |
| 161 | + let full_lock = open_file(&self.full)?; |
| 162 | + full_lock.lock_shared()?; |
| 163 | + Some(full_lock) |
| 164 | + } else { |
| 165 | + None |
| 166 | + }; |
| 167 | + |
| 168 | + self.guard = Some(UnitLockGuard { |
| 169 | + partial, |
| 170 | + _full: full, |
| 171 | + }); |
| 172 | + Ok(()) |
| 173 | + } |
| 174 | + |
| 175 | + pub fn downgrade(&mut self) -> CargoResult<()> { |
| 176 | + let guard = self |
| 177 | + .guard |
| 178 | + .as_ref() |
| 179 | + .context("guard was None while calling downgrade")?; |
| 180 | + |
| 181 | + // NOTE: |
| 182 | + // > Subsequent flock() calls on an already locked file will convert an existing lock to the new lock mode. |
| 183 | + // https://man7.org/linux/man-pages/man2/flock.2.html |
| 184 | + // |
| 185 | + // However, the `std::file::File::lock/lock_shared` is allowed to change this in the |
| 186 | + // future. So its probably up to us if we are okay with using this or if we want to use a |
| 187 | + // different interface to flock. |
| 188 | + guard.partial.lock_shared()?; |
| 189 | + |
| 190 | + Ok(()) |
| 191 | + } |
| 192 | +} |
| 193 | + |
| 194 | +fn open_file<T: AsRef<Path>>(f: T) -> CargoResult<File> { |
| 195 | + Ok(OpenOptions::new() |
| 196 | + .read(true) |
| 197 | + .create(true) |
| 198 | + .write(true) |
| 199 | + .append(true) |
| 200 | + .open(f)?) |
| 201 | +} |
| 202 | + |
| 203 | +fn all_dependency_units<'a>( |
| 204 | + build_runner: &'a BuildRunner<'a, '_>, |
| 205 | + unit: &Unit, |
| 206 | +) -> HashSet<&'a Unit> { |
| 207 | + fn inner<'a>( |
| 208 | + build_runner: &'a BuildRunner<'a, '_>, |
| 209 | + unit: &Unit, |
| 210 | + results: &mut HashSet<&'a Unit>, |
| 211 | + ) { |
| 212 | + for dep in build_runner.unit_deps(unit) { |
| 213 | + if results.insert(&dep.unit) { |
| 214 | + inner(&build_runner, &dep.unit, results); |
| 215 | + } |
| 216 | + } |
| 217 | + } |
| 218 | + |
| 219 | + let mut results = HashSet::new(); |
| 220 | + inner(build_runner, unit, &mut results); |
| 221 | + return results; |
| 222 | +} |
0 commit comments