Merge pull request #19 from JuliaLang/sd/nativeimage

add back option to build native image

Author: SimonDanisch

README.md

@@ -27,7 +27,7 @@ compile_package("Matcha", "relative/path/for_snooping.jl")
 revert()
 
 # Or if you simply want to get a native system image e.g. when you have downloaded the generic Julia install:
-build_clean_image()
+build_native_image()
 ```
 
 

src/PackageCompiler.jl

@@ -21,6 +21,7 @@ iswindows() && using WinRPM
 include("static_julia.jl")
 include("api.jl")
 include("snooping.jl")
+include("system_image.jl")
 
 const sysimage_binaries = (
     "sys.o", "sys.$(Libdl.dlext)", "sys.ji", "inference.o", "inference.ji"
@@ -36,6 +37,12 @@ function copy_system_image(src, dest, ignore_missing = false)
             ignore_missing && continue
             error("No file: $srcfile")
         end
+        if isfile(destfile)
+            if isfile(destfile * ".backup")
+                rm(destfile * ".backup", force = true)
+            end
+            mv(destfile, destfile * ".backup", remove_destination = true)
+        end
         info("Copying system image: $srcfile to $destfile")
         cp(srcfile, destfile, remove_destination = true)
     end
@@ -55,15 +62,7 @@ function replace_jl_sysimg(image_path, debug = false)
 end
 
 
-"""
-Builds a clean system image, similar to a fresh Julia install.
-Can also be used to build a native system image for a downloaded cross compiled julia binary.
-"""
-function build_clean_image(debug = false)
-    backup = sysimgbackup_folder()
-    build_sysimg(backup, joinpath(@__DIR__, "empty_userimg.jl"))
-    copy_system_image(backup, default_sysimg_path(debug))
-end
+
 
 """
 Reverts a forced compilation of the system image.
@@ -77,7 +76,7 @@ function revert(debug = false)
         copy_system_image(sysimg_backup, syspath)
     else
         warn("No backup found but restoring. Need to build a new system image from scratch")
-        build_clean_image(debug)
+        build_native_image(debug)
     end
 end
 
@@ -177,7 +176,7 @@ function compile_package(packages::Tuple{String, String}...; force = false, reus
 end
 
 
-export compile_package, revert, build_clean_image
+export compile_package, revert, build_native_image
 
 
 end # module

src/api.jl

@@ -52,3 +52,14 @@ function build_shared_lib(
         object = true, shared = true, executable = false, julialibs = true,
     )
 end
+
+
+"""
+Builds a clean system image, similar to a fresh Julia install.
+Can also be used to build a native system image for a downloaded cross compiled julia binary.
+"""
+function build_native_image(debug = false)
+    backup = sysimgbackup_folder()
+    compile_system_image(joinpath(backup, "sys"), "native")
+    copy_system_image(backup, default_sysimg_path(debug))
+end

src/system_image.jl

@@ -0,0 +1,57 @@
+function default_sysimg_path(debug = false)
+    ext = debug ? "sys-debug" : "sys"
+    if is_unix()
+        dirname(splitext(Libdl.dlpath(ext))[1])
+    else
+        joinpath(dirname(JULIA_HOME), "lib", "julia")
+    end
+end
+
+function compile_system_image(sysimg_path, cpu_target; debug = false)
+    # Enter base and setup some useful paths
+    base_dir = dirname(Base.find_source_file("sysimg.jl"))
+    cd(base_dir) do
+        julia = "julia"
+        cc = system_compiler()
+        # Ensure we have write-permissions to wherever we're trying to write to
+        try
+            touch("$sysimg_path.ji")
+        catch
+            err_msg =  "Unable to modify $sysimg_path.ji, ensure parent directory exists "
+            err_msg *= "and is writable; absolute paths work best.)"
+            error(err_msg)
+        end
+        # Start by building inference.{ji,o}
+        inference_path = joinpath(dirname(sysimg_path), "inference")
+        info("Building inference.o")
+        info("$julia -C $cpu_target --output-ji $inference_path.ji --output-o $inference_path.o coreimg.jl")
+        run(`$julia -C $cpu_target --output-ji $inference_path.ji --output-o $inference_path.o coreimg.jl`)
+
+        # Bootstrap off of that to create sys.{ji,o}
+        info("Building sys.o")
+        info("$julia -C $cpu_target --output-ji $sysimg_path.ji --output-o $sysimg_path.o -J $inference_path.ji --startup-file=no sysimg.jl")
+        run(`$julia -C $cpu_target --output-ji $sysimg_path.ji --output-o $sysimg_path.o -J $inference_path.ji --startup-file=no sysimg.jl`)
+
+        link_sysimg(sysimg_path, cc, debug)
+    end
+end
+
+# Link sys.o into sys.$(dlext)
+function link_sysimg(sysimg_path, cc = system_compiler(), debug = false)
+
+    julia_libdir = dirname(Libdl.dlpath(debug ? "libjulia-debug" : "libjulia"))
+
+    FLAGS = ["-L$julia_libdir"]
+
+    push!(FLAGS, "-shared")
+    push!(FLAGS, debug ? "-ljulia-debug" : "-ljulia")
+    if is_windows()
+        push!(FLAGS, "-lssp")
+    end
+
+    sysimg_file = "$sysimg_path.$(Libdl.dlext)"
+    info("Linking sys.$(Libdl.dlext)")
+    info("$cc $(join(FLAGS, ' ')) -o $sysimg_file $sysimg_path.o")
+    # Windows has difficulties overwriting a file in use so we first link to a temp file
+    run(`$cc $FLAGS -o $sysimg_file $sysimg_path.o`)
+end

test/runtests.jl

@@ -15,4 +15,10 @@ img_file = PackageCompiler.compile_package("Matcha", "UnicodeFun", force = false
     # Make sure we actually snooped stuff
     @test length(readlines(userimg)) > 700
     @test success(`julia -J $(img_file)`)
+    mktempdir() do dir
+        sysfile = joinpath(dir, "sys")
+        PackageCompiler.compile_system_image(sysfile, "native")
+        @test isfile(sysfile * ".o")
+        @test isfile(sysfile * ".$(Libdl.dlext)")
+    end
 end