Keep a Cache of Externally-Dependent Jobs

A more durable form of swiftlang#34218. Keep a side cache of externally-dependent
jobs for now. This ensures our pseudo-Jobs don't get prematurely
deallocated before the tracing machinery has had a chance to report the
structure of the Job graph.

rdar://70053563

Author: CodaFi

include/swift/Driver/Compilation.h

@@ -157,6 +157,16 @@ class Compilation {
 
   /// The Jobs which will be performed by this compilation.
   SmallVector<std::unique_ptr<const Job>, 32> Jobs;
+  // The Jobs which represent external actions performed by other drivers in
+  // the build graph.
+  //
+  // These Jobs are never scheduled into the build graph. This vector is
+  // populated by the routine that computes the set of incremental external
+  // dependencies that affect the current computation. Due to the way the
+  // Driver models multiple aspects of the incremental compilation scheduler
+  // by mapping to and from Jobs, it is necessary to lie and retain a set of
+  // pseudo-Jobs.
+  SmallVector<std::unique_ptr<const Job>, 32> ExternalJobs;
 
   /// The original (untranslated) input argument list.
   ///
@@ -348,7 +358,6 @@ class Compilation {
   UnwrappedArrayView<const Job> getJobs() const {
     return llvm::makeArrayRef(Jobs);
   }
-  Job *addJob(std::unique_ptr<Job> J);
 
   /// To send job list to places that don't truck in fancy array views.
   std::vector<const Job *> getJobsSimply() const {
@@ -515,6 +524,13 @@ class Compilation {
       const JobCollection &unsortedJobs,
       SmallVectorImpl<const Job *> &sortedJobs) const;
 
+private:
+  friend class Driver;
+  friend class PerformJobsState;
+
+  Job *addJob(std::unique_ptr<Job> J);
+  Job *addExternalJob(std::unique_ptr<Job> J);
+
 private:
   /// Perform all jobs.
   ///

lib/Driver/Compilation.cpp

@@ -1179,10 +1179,11 @@ namespace driver {
 
         // Cons up a fake `Job` to satisfy the incremental job tracing
         // code's internal invariants.
-        Job fakeJob(Comp.getDerivedOutputFileMap(), external);
+        const auto *externalJob = Comp.addExternalJob(
+            std::make_unique<Job>(Comp.getDerivedOutputFileMap(), external));
         auto subChanges =
             getFineGrainedDepGraph(forRanges).loadFromSwiftModuleBuffer(
-                &fakeJob, *buffer.get(), Comp.getDiags());
+                externalJob, *buffer.get(), Comp.getDiags());
 
         // If the incremental dependency graph failed to load, fall back to
         // treating this as plain external job.
@@ -1194,7 +1195,7 @@ namespace driver {
         for (auto *CMD :
              getFineGrainedDepGraph(forRanges)
                  .findJobsToRecompileWhenNodesChange(subChanges.getValue())) {
-          if (CMD == &fakeJob) {
+          if (CMD == externalJob) {
             continue;
           }
           ExternallyDependentJobs.push_back(CMD);
@@ -1717,6 +1718,12 @@ Job *Compilation::addJob(std::unique_ptr<Job> J) {
   return result;
 }
 
+Job *Compilation::addExternalJob(std::unique_ptr<Job> J) {
+  Job *result = J.get();
+  ExternalJobs.emplace_back(std::move(J));
+  return result;
+}
+
 static void checkForOutOfDateInputs(DiagnosticEngine &diags,
                                     const InputInfoMap &inputs) {
   for (const auto &inputPair : inputs) {