Merge pull request #173 from JuliaLang/sd-pgkcompile

get back compile_package

Author: SimonDanisch

REQUIRE

@@ -1,4 +1,3 @@
 julia 0.7
 ArgParse
-SnoopCompile
 @windows WinRPM

src/PackageCompiler.jl

@@ -1,6 +1,6 @@
 module PackageCompiler
 
-using Libdl, SnoopCompile
+using Libdl
 
 include("compiler_flags.jl")
 include("static_julia.jl")
@@ -111,15 +111,15 @@ use a toml instead.
 function compile_package(
         packages::Tuple{String, String}...;
         force = false, reuse = false, debug = false, cpu_target = nothing,
-        additional_packages = Symbol[]
+        additional_packages = Symbol[], verbose = false
     )
     userimg = sysimg_folder("precompile.jl")
     if !reuse
         snoop_userimg(userimg, packages...; additional_packages = additional_packages)
     end
     !isfile(userimg) && reuse && error("Nothing to reuse. Please run `compile_package(reuse = true)`")
     image_path = sysimg_folder()
-    build_sysimg(image_path, userimg, cpu_target=cpu_target)
+    build_sysimg(image_path, userimg, cpu_target=cpu_target, verbose = verbose)
     imgfile = joinpath(image_path, "sys.$(Libdl.dlext)")
     syspath = joinpath(default_sysimg_path(debug), "sys.$(Libdl.dlext)")
     if force

src/compiler_flags.jl

@@ -226,14 +226,20 @@ function run_julia(code::String; kw...)
 end
 
 function jl_command(flag, value)
-    isempty(value) && return ""
+    (value === nothing || isempty(value)) && return ""
     if is_short_flag(flag)
         string("-", flag, value)
     else
         string("--", flag, "=", value)
     end
 end
 
+
+function push_command!(cmd, flag, value)
+    command = jl_command(flag, value)
+    isempty(command) || push!(cmd.exec, command)
+end
+
 function julia_code_cmd(
         code::String, jl_options = Base.JLOptions();
         kw...
@@ -244,13 +250,12 @@ function julia_code_cmd(
     for (k, v) in kw
         jl_key = jl_option_key(k)
         flag = jl_options_to_flag[jl_key]
-        push!(cmd.exec, jl_command(flag, v))
+        push_command!(cmd, flag, v)
     end
     # add remaining commands from JLOptions
     for key in setdiff(keys(jl_options_to_flag), keys(kw))
         flag = jl_options_to_flag[key]
-        command = jl_command(flag, jl_option_value(jl_options, key))
-        isempty(command) || push!(cmd.exec, command)
+        push_command!(cmd, flag, jl_option_value(jl_options, key))
     end
     # for better debug, let's not make a tmp file which would get lost!
     file = sysimg_folder("run_julia_code.jl")

src/incremental.jl

@@ -4,6 +4,7 @@ Init basic C libraries
 function InitBase()
   """
   Base.__init__()
+  Sys.__init__() #fix https://github.com/JuliaLang/julia/issues/30479
   """
 end
 
@@ -17,17 +18,6 @@ function InitREPL()
   """
 end
 
-"""
-Fix for https://github.com/JuliaLang/julia/issues/30479
-"""
-function Fix30479()
-    """
-    _bindir = ccall(:jl_get_julia_bindir, Any, ())::String
-    @eval(Sys, BINDIR = \$(_bindir))
-    @eval(Sys, STDLIB = joinpath(\$_bindir, "..", "share", "julia", "stdlib", string('v', (VERSION.major), '.', VERSION.minor)))
-    """
-end
-
 function Include(path)
   """
   M = Module()
@@ -61,7 +51,7 @@ end
 The command to pass to julia --output-o, that runs the julia code in `path` during compilation.
 """
 function PrecompileCommand(path)
-    ExitHooksStart() * InitBase() * Fix30479() * InitREPL() * Include(path) * ExitHooksEnd()
+    ExitHooksStart() * InitBase() * InitREPL() * Include(path) * ExitHooksEnd()
 end
 
 
@@ -87,6 +77,7 @@ function compile_incremental(
         debug = false, cc_flags = nothing
     )
     precompiles = package_folder("incremental_precompile.jl")
+
     if snoopfile == nothing && precompile_file != nothing
         # we directly got a precompile_file
         isfile(precompile_file) || error("Need to pass an existing file to precompile_file. Found: $(repr(precompile_file))")
@@ -96,7 +87,7 @@ function compile_incremental(
     elseif snoopfile == nothing && precompile_file == nothing
         # reuse precompiles
     else
-        snoop(toml_path, snoopfile, precompiles)
+        snoop(nothing, toml_path, snoopfile, precompiles)
     end
     systemp = sysimg_folder("sys.a")
     sysout = sysimg_folder("sys.$(Libdl.dlext)")
@@ -138,19 +129,17 @@ function compile_incremental(packages::Symbol...; kw...)
         "compat" => Dict(),
     )
     precompiles_all = package_folder("incremental_precompile.jl")
-    open(precompiles_all, "w") do io
-        println(io, "# Precompile file for $(join(packages, " "))")
-    end
-    for package in packages
-        precompiles = package_folder(string(package), "incremental_precompile.jl")
-        toml, testfile = package_toml(package)
-        snoop(toml, testfile, precompiles)
-        pkg_toml = TOML.parsefile(toml)
-        merge!(finaltoml["deps"], get(pkg_toml, "deps", Dict()))
-        merge!(finaltoml["compat"], get(pkg_toml, "compat", Dict()))
-        open(precompiles_all, "a") do io
-            println(io)
-            write(io, read(precompiles))
+    open(precompiles_all, "w") do compile_io
+        println(compile_io, "# Precompile file for $(join(packages, " "))")
+        for package in packages
+            precompiles = package_folder(string(package), "incremental_precompile.jl")
+            toml, testfile = package_toml(package)
+            snoop(package, toml, testfile, precompiles)
+            pkg_toml = TOML.parsefile(toml)
+            merge!(finaltoml["deps"], get(pkg_toml, "deps", Dict()))
+            merge!(finaltoml["compat"], get(pkg_toml, "compat", Dict()))
+            println(compile_io)
+            write(compile_io, read(precompiles))
         end
     end
     toml = package_folder("Project.toml")

src/pkg.jl

@@ -25,54 +25,6 @@ function in_manifest(name::AbstractString, manifest::Dict = current_manifest())
 end
 
 
-"""
-    require_uninstalled(name::AbstractString, mod = Main)
-
-Loads a package, even if it isn't installed.
-Call this only in a precompile file used for incremental compilation!
-"""
-function require_uninstalled(name::AbstractString, mod = Main)
-    pkg = Base.PkgId(package_uuid(name), name)
-    psym = Symbol(name)
-    @eval mod begin
-        if !isdefined($mod, $(QuoteNode(psym)))
-            const $psym = Base.require($pkg)
-            # we need to call the __init__ because of
-            # https://github.com/JuliaLang/julia/issues/22910
-            if isdefined($psym, :__init__)
-                $psym.__init__()
-            end
-        else
-            @warn(string($name, " already defined"))
-        end
-    end
-end
-
-
-function extract_used_packages(file::String)
-    scope_regex = r"([\u00A0-\uFFFF\w_!´]*@?[\u00A0-\uFFFF\w_!´]+)\."
-    namespaces = unique(getindex.(eachmatch(scope_regex, read(file, String)), 1))
-    # only use names that are also in current manifest
-    return filter(in_manifest, namespaces)
-end
-
-
-"""
-Extracts using statements from a julia file.
-"""
-function extract_using(path, usings = Set{String}())
-    src = read(path, String)
-    regex = r"using ([\u00A0-\uFFFF\w_!´]+)(,[ \u00A0-\uFFFF\w_!´]+)?"
-    for match in eachmatch(regex, src)
-        push!(usings, match[1])
-        if match[2] !== nothing
-            pkgs = strip.(split(match[2], ',', keepempty = false))
-            union!(usings, pkgs)
-        end
-    end
-    return usings
-end
-
 #=
 genfile & create_project_from_require have been taken from the PR
 https://github.com/JuliaLang/PkgDev.jl/pull/144

src/snooping.jl

@@ -1,17 +1,19 @@
 using Pkg, Serialization
 
 
-function snoop(tomlpath, snoopfile, outputfile, reuse = false)
-    packages = extract_using(snoopfile)
+function snoop(package, tomlpath, snoopfile, outputfile, reuse = false)
+
     command = """
     using Pkg, PackageCompiler
     """
+
     if tomlpath != nothing
         command *= """
         Pkg.activate($(repr(tomlpath)))
         Pkg.instantiate()
         """
     end
+
     command *= """
     # let's wrap the snoop file in a try catch...
     # This way we still do some snooping even if there is an error in the tests!
@@ -24,29 +26,48 @@ function snoop(tomlpath, snoopfile, outputfile, reuse = false)
 
     # let's use a file in the PackageCompiler dir,
     # so it doesn't get lost if later steps fail
-    tmp_file = sysimg_folder("precompile_tmp.jl")
+    tmp_file = package_folder("precompile_tmp.jl")
     if !reuse
         run_julia(command, compile = "all", O = 0, g = 1, trace_compile = tmp_file)
     end
-    actually_used = extract_used_packages(tmp_file)
+    used_packages = Set{String}() # e.g. from test/REQUIRE
+    if package != nothing
+        push!(used_packages, string(package))
+    end
     if tomlpath != nothing
         # add toml packages, in case extract_used_packages misses a package
         deps = get(TOML.parsefile(tomlpath), "deps", Dict{String, Any}())
-        union!(actually_used, string.(keys(deps)))
+        union!(used_packages, string.(keys(deps)))
+    end
+    usings = if isempty(used_packages)
+        ""
+    else
+        packages = join(used_packages, ", ")
+        """
+        using $packages
+        for Mod in [$packages]
+            isdefined(Mod, :__init__) && Mod.__init__()
+        end
+        """
     end
 
     line_idx = 0; missed = 0
     open(outputfile, "w") do io
         println(io, """
-            # We need to use all used packages in the precompile file for maximum
-            # usage of the precompile statements.
-            # Since this can be any recursive dependency of the package we AOT compile,
-            # we decided to just use them without installing them. An added
-            # benefit is, that we can call __init__ this way more easily, since
-            # incremental sysimage compilation won't call __init__ on `using`
-            # https://github.com/JuliaLang/julia/issues/22910
-            using PackageCompiler
-            PackageCompiler.require_uninstalled.($(repr(actually_used)), (@__MODULE__,))
+        # We need to use all used packages in the precompile file for maximum
+        # usage of the precompile statements.
+        # Since this can be any recursive dependency of the package we AOT compile,
+        # we decided to just use them without installing them. An added
+        # benefit is, that we can call __init__ this way more easily, since
+        # incremental sysimage compilation won't call __init__ on `using`
+        # https://github.com/JuliaLang/julia/issues/22910
+        $usings
+        # bring recursive dependencies of used packages and standard libraries into namespace
+        for Mod in Base.loaded_modules_array()
+            if !Core.isdefined(@__MODULE__, nameof(Mod))
+                Core.eval(@__MODULE__, Expr(:const, Expr(:(=), nameof(Mod), Mod)))
+            end
+        end
         """)
         for line in eachline(tmp_file)
             line_idx += 1
@@ -61,17 +82,47 @@ function snoop(tomlpath, snoopfile, outputfile, reuse = false)
                 if expr.head != :incomplete
                     # after all this, we still need to wrap into try catch,
                     # since some anonymous symbols won't be found...
-                    println(io, "try;", line, "; catch e; @warn \"couldn't precompile statement $line_idx\" exception = e; end")
+                    println(io, "try;", line, "; catch e; @debug \"couldn't precompile statement $line_idx\" exception = e; end")
                 else
                     missed += 1
-                    @warn "Incomplete line in precompile file: $line"
+                    @debug "Incomplete line in precompile file: $line"
                 end
             catch e
                 missed += 1
-                @warn "Parse error in precompile file: $line" exception=e
+                @debug "Parse error in precompile file: $line" exception=e
             end
         end
     end
     @info "used $(line_idx - missed) out of $line_idx precompile statements"
     outputfile
 end
+
+
+"""
+    snoop_userimg(userimg, packages::Tuple{String, String}...)
+
+    Traces all function calls in packages and writes out `precompile` statements into the file `userimg`
+"""
+function snoop_userimg(userimg, packages::Tuple{String, String}...; additional_packages = Symbol[])
+    snooped_precompiles = map(packages) do package_snoopfile
+        package, snoopfile = package_snoopfile
+        module_name = Symbol(package)
+        toml, runtests = package_toml(module_name)
+        pkg_root = normpath(joinpath(dirname(runtests), ".."))
+        file2snoop = if isfile(pkg_root, snoopfile)
+            joinpath(pkg_root, snoopfile)
+        else
+            joinpath(pkg_root, snoopfile)
+        end
+        precompile_file = package_folder(package, "precompile.jl")
+        snoop(package, toml, file2snoop, precompile_file)
+        return precompile_file
+    end
+    # merge all of the temporary files into a single output
+    open(userimg, "w") do output
+        for path in snooped_precompiles
+            open(input -> write(output, input), path)
+        end
+    end
+    nothing
+end