Merge pull request #175 from JuliaLang/sd-precompile_toml

SimonDanisch · web-flow · commit 8987c26461e6 · 2019-01-31T17:47:02.000+01:00
precompile toml
diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl
@@ -110,12 +110,20 @@ use a toml instead.
 """
 function compile_package(
         packages::Tuple{String, String}...;
-        force = false, reuse = false, debug = false, cpu_target = nothing,
-        additional_packages = Symbol[], verbose = false
+        force = false, reuse = false, debug = false,
+        cpu_target = nothing, verbose = false
     )
     userimg = sysimg_folder("precompile.jl")
     if !reuse
-        snoop_userimg(userimg, packages...; additional_packages = additional_packages)
+        # TODO that's a pretty weak way to check that it's not a path...
+        ispackage = all(x-> !occursin(Base.Filesystem.path_separator, x), first.(packages))
+        isruntests = all(x-> x == "test/runtests.jl", last.(packages))
+        if ispackage && isruntests
+            snoop_packages(Symbol.(first.(packages))...; file = userimg)
+        else
+            ispackage || @warn "Giving path to package deprecated. Use Package name!"
+            isruntests || @warn "Giving a snoopfile is deprecated. Use runtests from package!"
+        end
     end
     !isfile(userimg) && reuse && error("Nothing to reuse. Please run `compile_package(reuse = true)`")
     image_path = sysimg_folder()
@@ -127,10 +135,12 @@ function compile_package(
             backup = syspath * ".packagecompiler_backup"
             isfile(backup) || mv(syspath, backup)
             cp(imgfile, syspath)
-            @info """Replaced system image successfully. Next start of julia will load the newly compiled system image.
-                If you encounter any errors with the new julia image, try `PackageCompiler.revert([debug = false])`."""
+            @info """
+            Replaced system image successfully. Next start of julia will load the newly compiled system image.
+            If you encounter any errors with the new julia image, try `PackageCompiler.revert([debug = false])`.
+            """
         catch e
-            @warn "An error occured while replacing sysimg files:" error=e
+            @warn "An error occured while replacing sysimg files:" error = e
             @info "Recovering old system image from backup"
             # if any file is missing in default system image, revert!
             if !isfile(syspath)
@@ -139,18 +149,14 @@ function compile_package(
             end
         end
     else
-        @info """Not replacing system image.
-            You can start julia with $(`julia -J $imgfile`) at a posix shell to load the compiled files."""
+        @info """
+        Not replacing system image.
+        You can start julia with $(`julia -J $imgfile`) at a posix shell to load the compiled files.
+        """
     end
     imgfile
 end
 
-function __init__()
-    if Base.julia_cmd().exec[2] != "-Cnative"
-        @warn "Your Julia system image is not compiled natively for this CPU architecture.\n" *
-            "Please run `PackageCompiler.force_native_image!()` for optimal Julia performance."
-    end
-end
 
 export compile_package, revert, force_native_image!, executable_ext, build_executable, build_shared_lib, static_julia, compile_incremental
 
diff --git a/src/incremental.jl b/src/incremental.jl
@@ -72,23 +72,10 @@ end
     To compile just a single package, see the simpler version  `compile_incremental(package::Symbol)`:
 """
 function compile_incremental(
-        toml_path::Union{String, Nothing}, snoopfile::Union{String, Nothing};
-        force = false, precompile_file = nothing, verbose = true,
+        toml_path::Union{String, Nothing}, precompiles::String;
+        force = false, verbose = true,
         debug = false, cc_flags = nothing
     )
-    precompiles = package_folder("incremental_precompile.jl")
-
-    if snoopfile == nothing && precompile_file != nothing
-        # we directly got a precompile_file
-        isfile(precompile_file) || error("Need to pass an existing file to precompile_file. Found: $(repr(precompile_file))")
-        if precompile_file != precompiles
-            cp(precompile_file, precompiles, force = true)
-        end
-    elseif snoopfile == nothing && precompile_file == nothing
-        # reuse precompiles
-    else
-        snoop(nothing, toml_path, snoopfile, precompiles)
-    end
     systemp = sysimg_folder("sys.a")
     sysout = sysimg_folder("sys.$(Libdl.dlext)")
     code = PrecompileCommand(precompiles)
@@ -104,7 +91,7 @@ end
 
 """
     compile_incremental(
-        package::Symbol;
+        packages::Symbol...;
         force = false, reuse = false, verbose = true,
         debug = false, cc_flags = nothing
     )
@@ -117,38 +104,7 @@ end
     For a more explicit version of compile_incremental, see:
     `compile_incremental(toml_path::String, snoopfile::String)`
 """
-function compile_incremental(package::Symbol; kw...)
-    toml, testfile = package_toml(package)
-    compile_incremental(toml, testfile; kw...)
-end
-
-
-function compile_incremental(packages::Symbol...; kw...)
-    finaltoml = Dict{Any, Any}(
-        "deps" => Dict(),
-        "compat" => Dict(),
-    )
-    precompiles_all = package_folder("incremental_precompile.jl")
-    open(precompiles_all, "w") do compile_io
-        println(compile_io, "# Precompile file for $(join(packages, " "))")
-        for package in packages
-            precompiles = package_folder(string(package), "incremental_precompile.jl")
-            toml, testfile = package_toml(package)
-            snoop(package, toml, testfile, precompiles)
-            pkg_toml = TOML.parsefile(toml)
-            merge!(finaltoml["deps"], get(pkg_toml, "deps", Dict()))
-            merge!(finaltoml["compat"], get(pkg_toml, "compat", Dict()))
-            println(compile_io)
-            write(compile_io, read(precompiles))
-        end
-    end
-    toml = package_folder("Project.toml")
-    finaltoml["name"] = "PackagesPrecompile"
-    open(toml, "w") do io
-        TOML.print(
-            io, finaltoml,
-            sorted = true, by = key-> (Types.project_key_order(key), key)
-        )
-    end
-    compile_incremental(toml, nothing; precompile_file = precompiles_all)
+function compile_incremental(pkg::Symbol, packages::Symbol...; kw...)
+    toml, precompile = snoop_packages(pkg, packages...)
+    compile_incremental(toml, precompile; kw...)
 end
diff --git a/src/snooping.jl b/src/snooping.jl
@@ -34,16 +34,15 @@ function snoop(package, tomlpath, snoopfile, outputfile, reuse = false)
     if package != nothing
         push!(used_packages, string(package))
     end
+    usings = ""
     if tomlpath != nothing
         # add toml packages, in case extract_used_packages misses a package
         deps = get(TOML.parsefile(tomlpath), "deps", Dict{String, Any}())
         union!(used_packages, string.(keys(deps)))
     end
-    usings = if isempty(used_packages)
-        ""
-    else
+    if !isempty(used_packages)
         packages = join(used_packages, ", ")
-        """
+        usings *= """
         using $packages
         for Mod in [$packages]
             isdefined(Mod, :__init__) && Mod.__init__()
@@ -98,31 +97,37 @@ function snoop(package, tomlpath, snoopfile, outputfile, reuse = false)
 end
 
 
-"""
-    snoop_userimg(userimg, packages::Tuple{String, String}...)
-
-    Traces all function calls in packages and writes out `precompile` statements into the file `userimg`
-"""
-function snoop_userimg(userimg, packages::Tuple{String, String}...; additional_packages = Symbol[])
-    snooped_precompiles = map(packages) do package_snoopfile
-        package, snoopfile = package_snoopfile
-        module_name = Symbol(package)
-        toml, runtests = package_toml(module_name)
-        pkg_root = normpath(joinpath(dirname(runtests), ".."))
-        file2snoop = if isfile(pkg_root, snoopfile)
-            joinpath(pkg_root, snoopfile)
-        else
-            joinpath(pkg_root, snoopfile)
+function snoop_packages(packages::Symbol...; file = package_folder("incremental_precompile.jl"))
+    finaltoml = Dict{Any, Any}(
+        "deps" => Dict(),
+        "compat" => Dict(),
+    )
+    toml_path = package_folder("Project.toml")
+    open(file, "w") do compile_io
+        println(compile_io, "# Precompile file for $(join(packages, " "))")
+        # make sure we have all packages from toml installed
+        println(compile_io, """
+        using Pkg
+        Pkg.activate($(repr(toml_path)))
+        Pkg.instantiate()
+        """)
+        for package in packages
+            precompiles = package_folder(string(package), "incremental_precompile.jl")
+            toml, testfile = package_toml(package)
+            snoop(package, toml, testfile, precompiles)
+            pkg_toml = TOML.parsefile(toml)
+            merge!(finaltoml["deps"], get(pkg_toml, "deps", Dict()))
+            merge!(finaltoml["compat"], get(pkg_toml, "compat", Dict()))
+            println(compile_io)
+            write(compile_io, read(precompiles))
         end
-        precompile_file = package_folder(package, "precompile.jl")
-        snoop(package, toml, file2snoop, precompile_file)
-        return precompile_file
     end
-    # merge all of the temporary files into a single output
-    open(userimg, "w") do output
-        for path in snooped_precompiles
-            open(input -> write(output, input), path)
-        end
+    finaltoml["name"] = "PackagesPrecompile"
+    open(toml_path, "w") do io
+        TOML.print(
+            io, finaltoml,
+            sorted = true, by = key-> (Types.project_key_order(key), key)
+        )
     end
-    nothing
+    return toml_path, file
 end
