fix lazy artifacts (#696)

Author: KristofferC

Project.toml

@@ -9,6 +9,7 @@ Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 RelocatableFolders = "05181044-ff0b-4ac5-8273-598c1e38db00"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]

examples/MyApp/Project.toml

@@ -10,4 +10,5 @@ Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 Example = "7876af07-990d-54b4-ab0e-23690620f79a"
 HelloWorldC_jll = "dca1746e-5efc-54fc-8249-22745bc95a49"
 LLVMExtra_jll = "dad2f222-ce93-54a1-a47d-0025e8a3acab"
+MKL_jll = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

examples/MyApp/src/MyApp.jl

@@ -5,7 +5,11 @@ using HelloWorldC_jll
 using Artifacts
 using Distributed
 using Random
-using LLVMExtra_jll
+if VERSION >= v"1.7.0"
+    using LLVMExtra_jll
+end
+
+using MKL_jll
 
 const myrand = rand()
 
@@ -81,10 +85,18 @@ function real_main()
     @eval @everywhere using Example
     @everywhere println(Example.domath(3))
 
-    if isfile(LLVMExtra_jll.libLLVMExtra_path)
-        println("LLVMExtra path: ok!")
+    if VERSION >= v"1.7.0"
+        if isfile(LLVMExtra_jll.libLLVMExtra_path)
+            println("LLVMExtra path: ok!")
+        else
+            println("LLVMExtra path: fail!")
+        end
+    end
+
+    if isfile(MKL_jll.libmkl_core_path)
+        println("MKL_jll path: ok!")
     else
-        println("LLVMExtra path: fail!")
+        println("MKL_jll path: fail!")
     end
     return
 end

src/PackageCompiler.jl

@@ -8,6 +8,7 @@ using Artifacts
 using LazyArtifacts
 using UUIDs: UUID, uuid1
 using RelocatableFolders
+using TOML
 
 export create_sysimage, create_app, create_library
 
@@ -1055,6 +1056,40 @@ function pretty_byte_str(size)
     return @sprintf("%.3f %s", bytes, Base._mem_units[mb])
 end
 
+# Copy pasted from Pkg since `collect_artifacts` doesn't allow lazy artifacts to get installed
+function _collect_artifacts(pkg_root::String; platform::Base.BinaryPlatforms.AbstractPlatform=HostPlatform(), include_lazy::Bool)
+    # Check to see if this package has an (Julia)Artifacts.toml
+    artifacts_tomls = Tuple{String,Base.TOML.TOMLDict}[]
+
+    for f in Artifacts.artifact_names
+        artifacts_toml = joinpath(pkg_root, f)
+        if isfile(artifacts_toml)
+            selector_path = joinpath(pkg_root, ".pkg", "select_artifacts.jl")
+
+            # If there is a dynamic artifact selector, run that in an appropriate sandbox to select artifacts
+            if isfile(selector_path)
+                # Despite the fact that we inherit the project, since the in-memory manifest
+                # has not been updated yet, if we try to load any dependencies, it may fail.
+                # Therefore, this project inheritance is really only for Preferences, not dependencies.
+                code = try Pkg.Operations.gen_build_code(selector_path; inherit_project=true)
+                catch e
+                    e isa MethodError || rethrow()
+                    Pkg.Operations.gen_build_code(selector_path)
+                end
+                select_cmd = Cmd(`$code $(Base.BinaryPlatforms.triplet(platform))`)
+                meta_toml = String(read(select_cmd))
+                push!(artifacts_tomls, (artifacts_toml, TOML.parse(meta_toml)))
+            else
+                # Otherwise, use the standard selector from `Artifacts`
+                artifacts = Pkg.Artifacts.select_downloadable_artifacts(artifacts_toml; platform, include_lazy)
+                push!(artifacts_tomls, (artifacts_toml, artifacts))
+            end
+            break
+        end
+    end
+    return artifacts_tomls
+end
+
 function bundle_artifacts(ctx, dest_dir; include_lazy_artifacts::Bool)
     pkgs = load_all_deps(ctx)
 
@@ -1076,7 +1111,7 @@ function bundle_artifacts(ctx, dest_dir; include_lazy_artifacts::Bool)
         pkg_source_path === nothing && continue
         bundled_artifacts_pkg = Pair{String, String}[]
         if isdefined(Pkg.Operations, :collect_artifacts)
-            for (artifacts_toml, artifacts) in Pkg.Operations.collect_artifacts(pkg_source_path; platform)
+            for (artifacts_toml, artifacts) in _collect_artifacts(pkg_source_path; platform, include_lazy=include_lazy_artifacts)
                 for (name, data) in artifacts
                     Pkg.ensure_artifact_installed(name, artifacts[name], artifacts_toml; platform)
                     hash = Base.SHA1(data["git-tree-sha1"])

test/runtests.jl

@@ -61,7 +61,7 @@ end
         for filter in filter_stdlibs
             tmp_app_source_dir = joinpath(tmp, "MyApp")
             cp(app_source_dir, tmp_app_source_dir)
-            create_app(tmp_app_source_dir, app_compiled_dir; incremental=incremental, force=true, filter_stdlibs=filter,
+            create_app(tmp_app_source_dir, app_compiled_dir; incremental=incremental, force=true, filter_stdlibs=filter, include_lazy_artifacts=true,
                        precompile_execution_file=joinpath(app_source_dir, "precompile_app.jl"),
                        executables=["MyApp" => "julia_main",
                                     "SecondApp" => "second_main",
@@ -111,7 +111,10 @@ end
             @test occursin("From worker 4:\t8", app_output)
             @test occursin("From worker 5:\t8", app_output)
 
-            @test occursin("LLVMExtra path: ok!", app_output)
+            if VERSION >= v"1.7.0"
+                @test occursin("LLVMExtra path: ok!", app_output)
+            end
+            @test occursin("MKL_jll path: ok!", app_output)
 
             # Test second app
             app_output = read(`$(app_path("SecondApp"))`, String)