DependenciesScanner: teach batch scanning mode to configure scanner using specified additional arguments

To help solving rdar://67079780, this change allows swift-driver to configure scanner using additional
arguments passed down via the batch input JSON file for each module under scanning.

Author: nkcsgexi

include/swift/AST/DiagnosticsCommon.def

@@ -185,5 +185,8 @@ WARNING(cross_imported_by_both_modules, none,
 ERROR(scanner_find_cycle, none,
       "dependency scanner detected dependency cycle: '%0'", (StringRef))
 
+ERROR(scanner_arguments_invalid, none,
+      "dependencies scanner cannot be configured with arguments: '%0'", (StringRef))
+
 #define UNDEFINE_DIAGNOSTIC_MACROS
 #include "DefineDiagnosticMacros.h"

lib/FrontendTool/FrontendTool.cpp

@@ -1751,7 +1751,8 @@ static void performEndOfPipelineActions(CompilerInstance &Instance) {
 /// \param Instance Will be reset after performIRGeneration when the verifier
 ///                 mode is NoVerify and there were no errors.
 /// \returns true on error
-static bool performCompile(CompilerInstance &Instance,
+static bool performCompile(CompilerInvocation &Invok,
+                           CompilerInstance &Instance,
                            ArrayRef<const char *> Args,
                            int &ReturnValue,
                            FrontendObserver *observer) {
@@ -1839,7 +1840,8 @@ static bool performCompile(CompilerInstance &Instance,
     if (batchScanInput.empty())
       return finishPipeline(scanDependencies(Instance));
     else
-      return finishPipeline(batchScanModuleDependencies(Instance,
+      return finishPipeline(batchScanModuleDependencies(Invok,
+                                                        Instance,
                                                         batchScanInput));
   }
 
@@ -2652,7 +2654,7 @@ int swift::performFrontend(ArrayRef<const char *> Args,
   }
 
   int ReturnValue = 0;
-  bool HadError = performCompile(*Instance, Args, ReturnValue, observer);
+  bool HadError = performCompile(Invocation, *Instance, Args, ReturnValue, observer);
 
   if (verifierEnabled) {
     DiagnosticEngine &diags = Instance->getDiags();

lib/FrontendTool/ScanDependencies.cpp

@@ -31,6 +31,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/YAMLTraits.h"
@@ -626,7 +627,6 @@ static bool diagnoseCycle(CompilerInstance &instance,
 
 static bool scanModuleDependencies(CompilerInstance &instance,
                                    StringRef moduleName,
-                                   StringRef arguments,
                                    bool isClang,
                                    StringRef outputPath) {
   ASTContext &ctx = instance.getASTContext();
@@ -685,25 +685,58 @@ static bool scanModuleDependencies(CompilerInstance &instance,
 bool swift::scanClangDependencies(CompilerInstance &instance) {
   return scanModuleDependencies(instance,
                                 instance.getMainModule()->getNameStr(),
-                                StringRef(),
                                 /*isClang*/true,
                                 instance.getInvocation().getFrontendOptions()
                                   .InputsAndOutputs.getSingleOutputFilename());
 }
 
-bool swift::batchScanModuleDependencies(CompilerInstance &instance,
-                                      llvm::StringRef batchInputFile) {
+bool swift::batchScanModuleDependencies(CompilerInvocation &invok,
+                                        CompilerInstance &instance,
+                                        llvm::StringRef batchInputFile) {
   (void)instance.getMainModule();
   llvm::BumpPtrAllocator alloc;
   llvm::StringSaver saver(alloc);
   auto results = parseBatchScanInputFile(instance.getASTContext(),
                                          batchInputFile, saver);
   if (!results.hasValue())
     return true;
+  auto &diags = instance.getDiags();
+  ForwardingDiagnosticConsumer FDC(diags);
+  // Keep track of all compiler instances we have created.
+  llvm::StringMap<std::unique_ptr<CompilerInstance>> subInstanceMap;
   for (auto &entry: *results) {
-    if (scanModuleDependencies(instance, entry.moduleName, entry.arguments,
-                               !entry.isSwift, entry.outputPath))
+    CompilerInstance *pInstance = nullptr;
+    if (entry.arguments.empty()) {
+      // Use the compiler's instance if no arguments are specified.
+      pInstance = &instance;
+    } else if (subInstanceMap.count(entry.arguments)) {
+      // Use the previously created instance if we've seen the arguments before.
+      pInstance = subInstanceMap[entry.arguments].get();
+    } else {
+      // Create a new instance by the arguments and save it in the map.
+      pInstance = subInstanceMap.insert({entry.arguments,
+        std::make_unique<CompilerInstance>()}).first->getValue().get();
+      SmallVector<const char*, 4> args;
+      llvm::cl::TokenizeGNUCommandLine(entry.arguments, saver, args);
+      CompilerInvocation subInvok = invok;
+      pInstance->addDiagnosticConsumer(&FDC);
+      if (subInvok.parseArgs(args, diags)) {
+        instance.getDiags().diagnose(SourceLoc(), diag::scanner_arguments_invalid,
+                                     entry.arguments);
+        return true;
+      }
+      if (pInstance->setup(subInvok)) {
+        instance.getDiags().diagnose(SourceLoc(), diag::scanner_arguments_invalid,
+                                     entry.arguments);
+        return true;
+      }
+    }
+    assert(pInstance);
+    // Scan using the chosen compiler instance.
+    if (scanModuleDependencies(*pInstance, entry.moduleName, !entry.isSwift,
+                               entry.outputPath)) {
       return true;
+    }
   }
   return false;
 }

lib/FrontendTool/ScanDependencies.h

@@ -17,10 +17,13 @@
 
 namespace swift {
 
+class CompilerInvocation;
 class CompilerInstance;
 
 /// Batch scan the dependencies for modules specified in \c batchInputFile.
-bool batchScanModuleDependencies(CompilerInstance &instance, llvm::StringRef batchInputFile);
+bool batchScanModuleDependencies(CompilerInvocation &invok,
+                                 CompilerInstance &instance,
+                                 llvm::StringRef batchInputFile);
 
 /// Scans the dependencies of the main module of \c instance.
 bool scanDependencies(CompilerInstance &instance);

test/ScanDependencies/Inputs/CHeaders/H.h

@@ -0,0 +1,5 @@
+#if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 110000
+#include "I.h"
+#endif
+
+void funcH(void);

test/ScanDependencies/Inputs/CHeaders/I.h

@@ -0,0 +1 @@
+void funcI(void);

test/ScanDependencies/Inputs/CHeaders/module.modulemap

@@ -27,3 +27,13 @@ module G {
   header "G.h"
   export *
 }
+
+module H {
+  header "H.h"
+  export *
+}
+
+module I {
+  header "I.h"
+  export *
+}

test/ScanDependencies/batch_module_scan_arguments.swift

@@ -0,0 +1,24 @@
+// RUN: %empty-directory(%t)
+// RUN: %empty-directory(%t/inputs)
+// RUN: %empty-directory(%t/outputs)
+// RUN: mkdir -p %t/clang-module-cache
+
+// RUN: echo "[{" > %/t/inputs/input.json
+// RUN: echo "\"clangModuleName\": \"H\"," >> %/t/inputs/input.json
+// RUN: echo "\"arguments\": \"-target x86_64-apple-macosx10.9\"," >> %/t/inputs/input.json
+// RUN: echo "\"output\": \"%/t/outputs/H.10.9.pcm.json\"" >> %/t/inputs/input.json
+// RUN: echo "}," >> %/t/inputs/input.json
+// RUN: echo "{" >> %/t/inputs/input.json
+// RUN: echo "\"clangModuleName\": \"H\"," >> %/t/inputs/input.json
+// RUN: echo "\"arguments\": \"-target x86_64-apple-macosx11.0\"," >> %/t/inputs/input.json
+// RUN: echo "\"output\": \"%/t/outputs/H.11.0.pcm.json\"" >> %/t/inputs/input.json
+// RUN: echo "}]" >> %/t/inputs/input.json
+
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 -batch-scan-input-file %/t/inputs/input.json
+
+// Check the contents of the JSON output
+// RUN: %FileCheck %s -check-prefix=CHECK-TEN < %t/outputs/H.10.9.pcm.json
+// RUN: %FileCheck %s -check-prefix=CHECK-ELEVEN < %t/outputs/H.11.0.pcm.json
+
+// CHECK-TEN: "clang": "I"
+// CHECK-ELEVEN-NOT: "clang": "I"