Auto merge of #34956 - nikomatsakis:incr-comp-o-files, r=mw

Enable reuse of `.o` files if nothing has changed

This PR completes a first "spike" for incremental compilation by enabling us to reuse `.o` files when nothing has changed. When in incr. mode, we will save `.o` files into the temporary directory, then copy them back out again if they are still valid. The code is still a bit rough but it does seem to work. =)

r? @michaelwoerister

Fixes #34036
Fixes #34037
Fixes #34038

Author: bors

src/librustc/dep_graph/dep_node.rs

@@ -9,6 +9,7 @@
 // except according to those terms.
 
 use std::fmt::Debug;
+use std::sync::Arc;
 
 macro_rules! try_opt {
     ($e:expr) => (
@@ -45,6 +46,10 @@ pub enum DepNode<D: Clone + Debug> {
     // in an extern crate.
     MetaData(D),
 
+    // Represents some artifact that we save to disk. Note that these
+    // do not have a def-id as part of their identifier.
+    WorkProduct(Arc<WorkProductId>),
+
     // Represents different phases in the compiler.
     CrateReader,
     CollectLanguageItems,
@@ -189,6 +194,11 @@ impl<D: Clone + Debug> DepNode<D> {
             TransCrate => Some(TransCrate),
             TransWriteMetadata => Some(TransWriteMetadata),
             LinkBinary => Some(LinkBinary),
+
+            // work product names do not need to be mapped, because
+            // they are always absolute.
+            WorkProduct(ref id) => Some(WorkProduct(id.clone())),
+
             Hir(ref d) => op(d).map(Hir),
             MetaData(ref d) => op(d).map(MetaData),
             CollectItem(ref d) => op(d).map(CollectItem),
@@ -229,3 +239,12 @@ impl<D: Clone + Debug> DepNode<D> {
         }
     }
 }
+
+/// A "work product" corresponds to a `.o` (or other) file that we
+/// save in between runs. These ids do not have a DefId but rather
+/// some independent path or string that persists between runs without
+/// the need to be mapped or unmapped. (This ensures we can serialize
+/// them even in the absence of a tcx.)
+#[derive(Clone, Debug, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable)]
+pub struct WorkProductId(pub String);
+

src/librustc/dep_graph/graph.rs

@@ -9,42 +9,65 @@
 // except according to those terms.
 
 use hir::def_id::DefId;
+use rustc_data_structures::fnv::FnvHashMap;
+use session::config::OutputType;
+use std::cell::{Ref, RefCell};
 use std::rc::Rc;
+use std::sync::Arc;
 
-use super::dep_node::DepNode;
+use super::dep_node::{DepNode, WorkProductId};
 use super::query::DepGraphQuery;
 use super::raii;
 use super::thread::{DepGraphThreadData, DepMessage};
 
 #[derive(Clone)]
 pub struct DepGraph {
-    data: Rc<DepGraphThreadData>
+    data: Rc<DepGraphData>
+}
+
+struct DepGraphData {
+    /// We send messages to the thread to let it build up the dep-graph
+    /// from the current run.
+    thread: DepGraphThreadData,
+
+    /// When we load, there may be `.o` files, cached mir, or other such
+    /// things available to us. If we find that they are not dirty, we
+    /// load the path to the file storing those work-products here into
+    /// this map. We can later look for and extract that data.
+    previous_work_products: RefCell<FnvHashMap<Arc<WorkProductId>, WorkProduct>>,
+
+    /// Work-products that we generate in this run.
+    work_products: RefCell<FnvHashMap<Arc<WorkProductId>, WorkProduct>>,
 }
 
 impl DepGraph {
     pub fn new(enabled: bool) -> DepGraph {
         DepGraph {
-            data: Rc::new(DepGraphThreadData::new(enabled))
+            data: Rc::new(DepGraphData {
+                thread: DepGraphThreadData::new(enabled),
+                previous_work_products: RefCell::new(FnvHashMap()),
+                work_products: RefCell::new(FnvHashMap())
+            })
         }
     }
 
     /// True if we are actually building a dep-graph. If this returns false,
     /// then the other methods on this `DepGraph` will have no net effect.
     #[inline]
     pub fn enabled(&self) -> bool {
-        self.data.enabled()
+        self.data.thread.enabled()
     }
 
     pub fn query(&self) -> DepGraphQuery<DefId> {
-        self.data.query()
+        self.data.thread.query()
     }
 
     pub fn in_ignore<'graph>(&'graph self) -> raii::IgnoreTask<'graph> {
-        raii::IgnoreTask::new(&self.data)
+        raii::IgnoreTask::new(&self.data.thread)
     }
 
     pub fn in_task<'graph>(&'graph self, key: DepNode<DefId>) -> raii::DepTask<'graph> {
-        raii::DepTask::new(&self.data, key)
+        raii::DepTask::new(&self.data.thread, key)
     }
 
     pub fn with_ignore<OP,R>(&self, op: OP) -> R
@@ -62,10 +85,84 @@ impl DepGraph {
     }
 
     pub fn read(&self, v: DepNode<DefId>) {
-        self.data.enqueue(DepMessage::Read(v));
+        self.data.thread.enqueue(DepMessage::Read(v));
     }
 
     pub fn write(&self, v: DepNode<DefId>) {
-        self.data.enqueue(DepMessage::Write(v));
+        self.data.thread.enqueue(DepMessage::Write(v));
+    }
+
+    /// Indicates that a previous work product exists for `v`. This is
+    /// invoked during initial start-up based on what nodes are clean
+    /// (and what files exist in the incr. directory).
+    pub fn insert_previous_work_product(&self, v: &Arc<WorkProductId>, data: WorkProduct) {
+        debug!("insert_previous_work_product({:?}, {:?})", v, data);
+        self.data.previous_work_products.borrow_mut()
+                                        .insert(v.clone(), data);
+    }
+
+    /// Indicates that we created the given work-product in this run
+    /// for `v`. This record will be preserved and loaded in the next
+    /// run.
+    pub fn insert_work_product(&self, v: &Arc<WorkProductId>, data: WorkProduct) {
+        debug!("insert_work_product({:?}, {:?})", v, data);
+        self.data.work_products.borrow_mut()
+                               .insert(v.clone(), data);
     }
+
+    /// Check whether a previous work product exists for `v` and, if
+    /// so, return the path that leads to it. Used to skip doing work.
+    pub fn previous_work_product(&self, v: &Arc<WorkProductId>) -> Option<WorkProduct> {
+        self.data.previous_work_products.borrow()
+                                        .get(v)
+                                        .cloned()
+    }
+
+    /// Access the map of work-products created during this run. Only
+    /// used during saving of the dep-graph.
+    pub fn work_products(&self) -> Ref<FnvHashMap<Arc<WorkProductId>, WorkProduct>> {
+        self.data.work_products.borrow()
+    }
+}
+
+/// A "work product" is an intermediate result that we save into the
+/// incremental directory for later re-use. The primary example are
+/// the object files that we save for each partition at code
+/// generation time.
+///
+/// Each work product is associated with a dep-node, representing the
+/// process that produced the work-product. If that dep-node is found
+/// to be dirty when we load up, then we will delete the work-product
+/// at load time. If the work-product is found to be clean, then we
+/// will keep a record in the `previous_work_products` list.
+///
+/// In addition, work products have an associated hash. This hash is
+/// an extra hash that can be used to decide if the work-product from
+/// a previous compilation can be re-used (in addition to the dirty
+/// edges check).
+///
+/// As the primary example, consider the object files we generate for
+/// each partition. In the first run, we create partitions based on
+/// the symbols that need to be compiled. For each partition P, we
+/// hash the symbols in P and create a `WorkProduct` record associated
+/// with `DepNode::TransPartition(P)`; the hash is the set of symbols
+/// in P.
+///
+/// The next time we compile, if the `DepNode::TransPartition(P)` is
+/// judged to be clean (which means none of the things we read to
+/// generate the partition were found to be dirty), it will be loaded
+/// into previous work products. We will then regenerate the set of
+/// symbols in the partition P and hash them (note that new symbols
+/// may be added -- for example, new monomorphizations -- even if
+/// nothing in P changed!). We will compare that hash against the
+/// previous hash. If it matches up, we can reuse the object file.
+#[derive(Clone, Debug, RustcEncodable, RustcDecodable)]
+pub struct WorkProduct {
+    /// Extra hash used to decide if work-product is still suitable;
+    /// note that this is *not* a hash of the work-product itself.
+    /// See documentation on `WorkProduct` type for an example.
+    pub input_hash: u64,
+
+    /// Saved files associated with this CGU
+    pub saved_files: Vec<(OutputType, String)>,
 }

src/librustc/dep_graph/mod.rs

@@ -20,7 +20,9 @@ mod visit;
 
 pub use self::dep_tracking_map::{DepTrackingMap, DepTrackingMapConfig};
 pub use self::dep_node::DepNode;
+pub use self::dep_node::WorkProductId;
 pub use self::graph::DepGraph;
+pub use self::graph::WorkProduct;
 pub use self::query::DepGraphQuery;
 pub use self::visit::visit_all_items_in_krate;
 pub use self::raii::DepTask;

src/librustc/session/config.rs

@@ -61,7 +61,7 @@ pub enum DebugInfoLevel {
     FullDebugInfo,
 }
 
-#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)]
 pub enum OutputType {
     Bitcode,
     Assembly,
@@ -105,6 +105,17 @@ impl OutputType {
             OutputType::DepInfo => "dep-info",
         }
     }
+
+    pub fn extension(&self) -> &'static str {
+        match *self {
+            OutputType::Bitcode => "bc",
+            OutputType::Assembly => "s",
+            OutputType::LlvmAssembly => "ll",
+            OutputType::Object => "o",
+            OutputType::DepInfo => "d",
+            OutputType::Exe => "",
+        }
+    }
 }
 
 #[derive(Clone)]
@@ -215,15 +226,7 @@ impl OutputFilenames {
                      flavor: OutputType,
                      codegen_unit_name: Option<&str>)
                      -> PathBuf {
-        let extension = match flavor {
-            OutputType::Bitcode => "bc",
-            OutputType::Assembly => "s",
-            OutputType::LlvmAssembly => "ll",
-            OutputType::Object => "o",
-            OutputType::DepInfo => "d",
-            OutputType::Exe => "",
-        };
-
+        let extension = flavor.extension();
         self.temp_path_ext(extension, codegen_unit_name)
     }
 

src/librustc/session/mod.rs

@@ -80,7 +80,7 @@ pub struct Session {
     // forms a unique global identifier for the crate. It is used to allow
     // multiple crates with the same name to coexist. See the
     // trans::back::symbol_names module for more information.
-    pub crate_disambiguator: Cell<ast::Name>,
+    pub crate_disambiguator: RefCell<token::InternedString>,
     pub features: RefCell<feature_gate::Features>,
 
     /// The maximum recursion limit for potentially infinitely recursive
@@ -106,6 +106,9 @@ pub struct Session {
 }
 
 impl Session {
+    pub fn local_crate_disambiguator(&self) -> token::InternedString {
+        self.crate_disambiguator.borrow().clone()
+    }
     pub fn struct_span_warn<'a, S: Into<MultiSpan>>(&'a self,
                                                     sp: S,
                                                     msg: &str)
@@ -438,7 +441,7 @@ pub fn build_session_(sopts: config::Options,
         plugin_attributes: RefCell::new(Vec::new()),
         crate_types: RefCell::new(Vec::new()),
         dependency_formats: RefCell::new(FnvHashMap()),
-        crate_disambiguator: Cell::new(token::intern("")),
+        crate_disambiguator: RefCell::new(token::intern("").as_str()),
         features: RefCell::new(feature_gate::Features::new()),
         recursion_limit: Cell::new(64),
         next_node_id: Cell::new(1),

src/librustc/ty/context.rs

@@ -504,7 +504,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
 
     pub fn crate_disambiguator(self, cnum: ast::CrateNum) -> token::InternedString {
         if cnum == LOCAL_CRATE {
-            self.sess.crate_disambiguator.get().as_str()
+            self.sess.local_crate_disambiguator()
         } else {
             self.sess.cstore.crate_disambiguator(cnum)
         }

src/librustc/util/fs.rs

@@ -10,6 +10,8 @@
 
 use std::path::{self, Path, PathBuf};
 use std::ffi::OsString;
+use std::fs;
+use std::io;
 
 // Unfortunately, on windows, it looks like msvcrt.dll is silently translating
 // verbatim paths under the hood to non-verbatim paths! This manifests itself as
@@ -53,3 +55,15 @@ pub fn fix_windows_verbatim_for_gcc(p: &Path) -> PathBuf {
         _ => p.to_path_buf(),
     }
 }
+
+/// Copy `p` into `q`, preferring to use hard-linking if possible. If
+/// `q` already exists, it is removed first.
+pub fn link_or_copy<P: AsRef<Path>, Q: AsRef<Path>>(p: P, q: Q) -> io::Result<()> {
+    let p = p.as_ref();
+    let q = q.as_ref();
+    if q.exists() {
+        try!(fs::remove_file(&q));
+    }
+    fs::hard_link(p, q)
+        .or_else(|_| fs::copy(p, q).map(|_| ()))
+}