Merge pull request #4 from SimonDanisch/sd/cleanup

SimonDanisch · web-flow · commit fe53c05afc77 · 2017-12-07T14:55:49.000+01:00
fixes, multiple packages, more modularity
diff --git a/.travis.yml b/.travis.yml
@@ -6,11 +6,16 @@ os:
 julia:
   - 0.6
   - nightly
+matrix:
+  allow_failures:
+    - julia: nightly
+    
 notifications:
   email: false
 git:
   depth: 99999999
 
+
 ## uncomment the following lines to allow failures on nightly julia
 ## (tests will run but not make your overall status red)
 #matrix:
diff --git a/appveyor.yml b/appveyor.yml
@@ -2,8 +2,6 @@ environment:
   matrix:
   - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe"
   - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
-  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe"
-  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
 
 ## uncomment the following lines to allow failures on nightly julia
 ## (tests will run but not make your overall status red)
diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl
@@ -9,16 +9,14 @@ const sysimage_binaries = (
     "sys.o", "sys.$(Libdl.dlext)", "sys.ji", "inference.o", "inference.ji"
 )
 
-function snoop(path, compilationfile, csv, reuse)
+function snoop(path, compilationfile, csv)
     cd(@__DIR__)
     # Snoop compiler can't handle the path as a variable, so we just create a file
-    if !reuse
-        open(joinpath("snoopy.jl"), "w") do io
-            println(io, "include(\"$(escape_string(path))\")")
-        end
-        SnoopCompile.@snoop csv begin
-            include("snoopy.jl")
-        end
+    open(joinpath("snoopy.jl"), "w") do io
+        println(io, "include(\"$(escape_string(path))\")")
+    end
+    SnoopCompile.@snoop csv begin
+        include("snoopy.jl")
     end
     data = SnoopCompile.read(csv)
     pc = SnoopCompile.parcel(reverse!(data[2]))
@@ -40,12 +38,15 @@ function snoop(path, compilationfile, csv, reuse)
                 # replace `_` for free parameters, which print out a warning otherwise
                 ln = replace(ln, delims, s"\1XXX\2")
                 # only print out valid lines
-                # TODO figure out why some precompile statements have undefined free variables in there
+                # TODO figure out the actual problems and why snoop compile emits invalid code
                 try
-                    eval(tmp_mod, parse(ln))
-                    println(io, ln)
+                    parse(ln) # parse to make sure expression is parsing without error
+                    # wrap in try catch to catch problematic code emitted by SnoopCompile
+                    # without interupting the whole precompilation
+                    # (usually, SnoopCompile emits 1% erroring statements)
+                    println(io, "try\n    ", ln, "\nend")
                 catch e
-                    warn("Not emitted: ", ln)
+                    warn("Not emitted because code couldn't parse: ", ln)
                 end
             end
         end
@@ -95,40 +96,84 @@ function get_root_dir(path)
     end
 end
 
+function snoop_package(package::String, rel_snoop_file, sysimg_tmp, reuse)
+    precompile_file = joinpath(sysimg_tmp, "precompile.jl")
+
+end
+
+function sysimg_folder(files...)
+    base_path = normpath(abspath(joinpath(@__DIR__, "..", "sysimg")))
+    isdir(base_path) || mkpath(base_path)
+    normpath(abspath(joinpath(base_path, files...)))
+end
 
-function compile_package(package, force = false, reuse = false; debug = false)
-    realpath = if ispath(package)
-        normpath(abspath(package))
-    else
-        Pkg.dir(package)
+function sysimgbackup_folder(files...)
+    backup = sysimg_folder("backup")
+    isdir(backup) || mkpath(backup)
+    sysimg_folder("backup", files...)
+end
+
+
+function package_folder(package...)
+    packages = normpath(abspath(joinpath(@__DIR__, "..", "packages")))
+    isdir(packages) || mkpath(packages)
+    normpath(abspath(joinpath(packages, package...)))
+end
+
+
+function compile_package(packages...; kw_args...)
+    args = map(packages) do package
+        # If no explicit path to a seperate precompile file, use runtests
+        isa(package, String) && return (package, "test/runtests.jl")
+        isa(package, Tuple{String, String}) && return package
+        error("Unrecognized package. Use `packagename::String`, or (packagename::String, rel_path_to_testfile::String). Found: $package")
     end
-    testroot = joinpath(realpath, "test")
-    sysimg_tmp = normpath(joinpath(@__DIR__, "..", "sysimg_tmp", get_root_dir(realpath)))
-    sysimg_backup = joinpath(@__DIR__, "..", "sysimg_backup")
-    isdir(sysimg_backup) || mkpath(sysimg_tmp)
-    isdir(sysimg_tmp) || mkpath(sysimg_tmp)
-    precompile_file = joinpath(sysimg_tmp, "precompile.jl")
-    snoop(
-        joinpath(testroot, "runtests.jl"),
-        precompile_file,
-        joinpath(sysimg_tmp, "snooped.csv"),
-        reuse
-    )
-    build_sysimg(joinpath(sysimg_tmp, "sys"), "native", precompile_file)
+    compile_package(args...; kw_args...)
+end
+
+function snoop_userimg(userimg, packages::Tuple{String, String}...)
+    snooped_precompiles = map(packages) do package_snoopfile
+        package, snoopfile = package_snoopfile
+        abs_package_path = if ispath(package)
+            normpath(abspath(package))
+        else
+            Pkg.dir(package)
+        end
+        file2snoop = normpath(abspath(joinpath(abs_package_path, snoopfile)))
+        package = package_folder(get_root_dir(abs_package_path))
+        isdir(package) || mkpath(package)
+        precompile_file = joinpath(package, "precompile.jl")
+        snoop(
+            file2snoop,
+            precompile_file,
+            joinpath(package, "snooped.csv")
+        )
+        precompile_file
+    end
+    open(userimg, "w") do io
+        for path in snooped_precompiles
+            write(io, open(read, path))
+            println(io)
+        end
+    end
+    userimg
+end
+
+function compile_package(packages::Tuple{String, String}...; force = false, reuse = false, debug = false)
+    userimg = sysimg_folder("precompile.jl")
+    if !reuse
+        snoop_userimg(userimg, packages...)
+    end
+    !isfile(userimg) && reuse && error("Nothing to reuse. Please run `compile_package(reuse = true)`")
+    image_path = sysimg_folder("sys")
+    build_sysimg(image_path, "native", userimg)
     if force
         try
-            syspath = default_sysimg_path(debug)
-            for file in sysimage_binaries
-                # backup
-                bfile = joinpath(sysimg_backup, file)
-                sfile = joinpath(dirname(syspath), file)
-                if !isfile(bfile) # use the one that is already there
-                    mv(sfile, bfile, remove_destination = true)
-                else
-                    mv(sfile, sfile*".old", remove_destination = true) # remove so we don't overwrite (seems to be problematic on windows)
-                end
-                mv(joinpath(sysimg_tmp, file), sfile, remove_destination = false)
-            end
+            replace_jl_sysimg(debug)
+            info(
+                "Replaced system image successfully. Next start of julia will load the newly compiled system image.
+                If you encounter any errors with the new julia image, try `PackageCompiler.revert([debug = false])`"
+            )
         catch e
             warn("An error has occured while replacing sysimg files:")
             warn(e)
@@ -138,9 +183,26 @@ function compile_package(package, force = false, reuse = false; debug = false)
     else
         info("""
             Not replacing system image.
-            You can start julia with julia -J $(joinpath(sysimg_tmp, "sys")) to load the compiled files.
+            You can start julia with julia -J $(image_path) to load the compiled files.
         """)
     end
 end
 
+
+function replace_jl_sysimg(debug = false)
+    syspath = default_sysimg_path(debug)
+    for file in sysimage_binaries
+        # backup
+        bfile = sysimgbackup_folder(file)
+        sfile = joinpath(dirname(syspath), file)
+        if !isfile(bfile) # use the one that is already there
+            mv(sfile, bfile, remove_destination = true)
+        else
+            mv(sfile, sfile*".old", remove_destination = true) # remove so we don't overwrite (seems to be problematic on windows)
+        end
+        mv(joinpath(sysimg_tmp, file), sfile, remove_destination = false)
+    end
+end
+
+
 end # module
diff --git a/src/build_sysimg.jl b/src/build_sysimg.jl
@@ -53,12 +53,7 @@ current processor. Include the user image file given by `userimg_path`, which sh
 directives such as `using MyPackage` to include that package in the new system image. New
 system image will not replace an older image unless `force` is set to true.
 """
-function build_sysimg(sysimg_path, cpu_target = "native", userimg_path = nothing; debug=false)
-
-    # Canonicalize userimg_path before we enter the base_dir
-    if userimg_path !== nothing
-        userimg_path = abspath(userimg_path)
-    end
+function build_sysimg(sysimg_path, cpu_target, userimg_path; debug = false)
     # Enter base and setup some useful paths
     base_dir = dirname(Base.find_source_file("sysimg.jl"))
     cd(base_dir) do
@@ -74,15 +69,13 @@ function build_sysimg(sysimg_path, cpu_target = "native", userimg_path = nothing
         end
 
         # Copy in userimg.jl if it exists
-        if userimg_path !== nothing
-            if !isfile(userimg_path)
-                error("$userimg_path is not found, ensure it is an absolute path.")
-            end
-            if isfile("userimg.jl")
-                error("$base_dir/userimg.jl already exists, delete manually to continue.")
-            end
-            cp(userimg_path, "userimg.jl")
+        if !isfile(userimg_path)
+            error("$userimg_path is not found, ensure it is an absolute path.")
+        end
+        if isfile("userimg.jl")
+            error("$base_dir/userimg.jl already exists, delete manually to continue.")
         end
+        cp(userimg_path, "userimg.jl")
         try
             # Start by building inference.{ji,o}
             inference_path = joinpath(dirname(sysimg_path), "inference")
@@ -99,7 +92,7 @@ function build_sysimg(sysimg_path, cpu_target = "native", userimg_path = nothing
 
         finally
             # Cleanup userimg.jl
-            if userimg_path !== nothing && isfile("userimg.jl")
+            if isfile("userimg.jl")
                 rm("userimg.jl")
             end
         end
diff --git a/test/REQUIRE b/test/REQUIRE
@@ -1 +1,2 @@
 Matcha
+UnicodeFun
diff --git a/test/runtests.jl b/test/runtests.jl
@@ -3,5 +3,15 @@ using Base.Test
 
 # If this works without error we should be in pretty good shape!
 # This command will use the runtest.jl of Matcha to find out what functions to precompile!
-PackageCompiler.compile_package("Matcha", false) # false to not force overwriting julia's current system image
+PackageCompiler.compile_package("Matcha", "UnicodeFun", force = false, reuse = false) # false to not force overwriting julia's current system image
+# build again, with resuing the snoop file
+PackageCompiler.compile_package("Matcha", "UnicodeFun", force = false, reuse = true)
 # TODO test revert - I suppose i wouldn't have enough rights on travis to move around dll's?
+
+@testset "basic tests" begin
+    @test isfile(PackageCompiler.sysimg_folder("sys.$(Libdl.dlext)"))
+    userimg = PackageCompiler.sysimg_folder("precompile.jl")
+    @test isfile(userimg)
+    # Make sure we actually snooped stuff
+    @test length(readlines(userimg)) > 700
+end
