Merge pull request #20 from JuliaLang/sd/tests

add tests and build_executable

Author: SimonDanisch

.gitignore

@@ -14,3 +14,4 @@ sysimg
 *.dll
 hello
 hello.exe
+packages

README.md

@@ -28,6 +28,14 @@ revert()
 
 # Or if you simply want to get a native system image e.g. when you have downloaded the generic Julia install:
 build_native_image()
+
+# building an executable
+
+build_executable(
+    "hello.jl", # julia file containing a julia main, e.g. like examples/hello.jl
+    snoopfile = "call_functions.jl", # julia file that calls functions that you want to make sure to have precompiled [optional]
+    builddir = "folder/you/want/the/build/artifacts" # that's where hello.exe will end up
+)
 ```
 
 
@@ -49,6 +57,7 @@ build_native_image()
 
 # Static Julia Compiler
 
+
 Building shared libraries and executables from Julia code.
 
 Run `juliac.jl -h` for help:
@@ -110,10 +119,7 @@ examples:
 
 ### Notes
 
-1. The `juliac.jl` script uses the `ArgParse` package, make sure it is installed.
-
-3. On Windows install `Cygwin` and the `mingw64-x86_64-gcc-core` package, see:\
-   https://github.com/JuliaLang/julia/blob/master/README.windows.md
+1. The `juliac.jl` script is located in the PackageCompiler root folder (Pkg.dir("PackageCompiler"))
 
 2. A shared library containing the system image `libhello.so`, and a
    driver binary `hello` are created in the `builddir` directory.
@@ -122,25 +128,26 @@ examples:
 ```
    $ ./hello
    hello, world
-   f() = -0.37549581296986956
-       ┌─────────────────────────────────────────────────┐
-   100 │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠎│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡰⠁⠀│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣠⠊⠀⠀⠀│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠔⠁⠀⠀⠀⠀│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠔⠁⠀⠀⠀⠀⠀⠀│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡠⠊⠀⠀⠀⠀⠀⠀⠀⠀│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡠⠊⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡠⠊⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠤⠊⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⠔⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠤⠊⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⠤⠒⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
-       │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⡠⠔⠊⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
-       │⠀⠀⠀⠀⠀⠀⢀⣀⠤⠔⠊⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
-     0 │⣀⠤⠤⠔⠒⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
-       └─────────────────────────────────────────────────┘
-       1                                       10
+   sin(0.0) = 0.0
+         ┌────────────────────────────────────────┐
+       1 │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡠⠔⠉⠉⠉⠉⠉⠒⢄⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
+         │⠀⠀⠀⠀⠀⠀⠀⠀⢠⠔⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠑⢄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
+         │⠀⠀⠀⠀⠀⠀⠀⡔⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠱⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
+         │⠀⠀⠀⠀⠀⡠⠊⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠢⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
+         │⠀⠀⠀⢀⠎⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⢆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
+         │⠀⠀⡠⠃⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠣⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
+         │⠀⡔⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠑⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀│
+         │⠮⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠬⢦⠤⠤⠤⠤⠤⠤⠤⠄│
+         │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠣⡀⠀⠀⠀⠀⠀⠀│
+         │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠘⢄⠀⠀⠀⠀⠀│
+         │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⢢⠀⠀⠀⠀│
+         │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠑⢄⠀⠀│
+         │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠣⡀│
+         │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈│
+      -1 │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│
+         └────────────────────────────────────────┘
+         1                                       10
+
 ```
 
 ## Under the hood

examples/hello.jl

@@ -1,13 +1,11 @@
 module Hello
 
 using UnicodePlots
-using Distributions
-f() = rand(Normal())
 
 Base.@ccallable function julia_main(ARGS::Vector{String})::Cint
     println("hello, world")
-    @show f()
-    println(lineplot(1:10, (1:10).^2))
+    @show sin(0.0)
+    println(lineplot(1:10, sin.(linspace(0, 4, 10))))
     return 0
 end
 

examples/program.c

@@ -16,28 +16,11 @@ JULIA_DEFINE_FAST_TLS()
 extern int julia_main(jl_array_t*);
 
 // main function (windows UTF16 -> UTF8 argument conversion code copied from julia's ui/repl.c)
-#ifndef _OS_WINDOWS_
 int main(int argc, char *argv[])
 {
     int retcode;
     int i;
     uv_setup_args(argc, argv); // no-op on Windows
-#else
-int wmain(int argc, wchar_t *wargv[], wchar_t *envp[])
-{
-    int retcode;
-    int i;
-    char **argv;
-    for (i = 0; i < argc; i++) { // convert the command line to UTF8
-        wchar_t *warg = argv[i];
-        size_t len = WideCharToMultiByte(CP_UTF8, 0, warg, -1, NULL, 0, NULL, NULL);
-        if (!len) return 1;
-        char *arg = (char*)alloca(len);
-        if (!WideCharToMultiByte(CP_UTF8, 0, warg, -1, arg, len, NULL, NULL)) return 1;
-        argv[i] = arg;
-    }
-#endif
-
     // initialization
     libsupport_init();
     // jl_options.compile_enabled = JL_OPTIONS_COMPILE_OFF;

juliac.jl

@@ -1,10 +1,6 @@
-## Assumptions:
-## 1. gcc / x86_64-w64-mingw32-gcc is available and is in path
-## 2. Package ArgParse is installed
-
 using PackageCompiler, ArgParse
 
-function main(args)
+Base.@ccallable function julia_main(args::Vector{String})::Cint
 
     s = ArgParseSettings("Static Julia Compiler",
                          version = "$(basename(@__FILE__)) version 0.7-DEV",
@@ -128,4 +124,6 @@ function main(args)
     )
 end
 
-main(ARGS)
+if get(ENV, "COMPILE_STATIC", "false") == "false"
+    julia_main(ARGS)
+end

src/PackageCompiler.jl

@@ -7,10 +7,12 @@ const julia_v07 = VERSION > v"0.7-"
 if julia_v07
     using Libdl
     import Sys: iswindows, isunix, isapple
+    const contains07 = contains
 else
     const iswindows = is_windows
     const isunix = is_unix
     const isapple = is_apple
+    contains07(str, reg) = ismatch(reg, str)
 end
 
 using SnoopCompile
@@ -176,7 +178,16 @@ function compile_package(packages::Tuple{String, String}...; force = false, reus
 end
 
 
-export compile_package, revert, build_native_image
 
 
+function __init__()
+    if Base.julia_cmd().exec[2] != "-Cnative"
+        warn("Your Julia system image is not compiled natively for this CPU architecture.
+        Please run `PackageCompiler.build_native_image()` for optimal Julia performance"
+        )
+    end
+end
+
+export compile_package, revert, build_native_image, executable_ext, build_executable
+
 end # module

src/api.jl

@@ -35,7 +35,7 @@ end
 function build_shared_lib(
         library, library_name;
         verbose = false, quiet = false,
-        cpu_target = "native", optimize = nothing, debug = nothing,
+        cpu_target = nothing, optimize = nothing, debug = nothing,
         inline = nothing, check_bounds = nothing, math_mode = nothing
     )
     julia_compile(
@@ -53,13 +53,66 @@ function build_shared_lib(
     )
 end
 
+"""
+    build_executable(
+        library,
+        library_name = splitext(basename(library))[1],
+        cprogram = joinpath(@__DIR__, "..", "examples", "program.c");
+
+        snoopfile = nothing, builddir = "build",
+        verbose = false, quiet = false,
+        cpu_target = nothing, optimize = nothing, debug = nothing,
+        inline = nothing, check_bounds = nothing, math_mode = nothing
+    )
+
+    `library` needs to be a julia file containing a julia main, e.g. like examples/hello.jl
+    `snoopfile` is optional and can be julia file that calls functions that you want to make sure to have precompiled
+    `builddir` is where library_name.exe and shared libraries will end up
+"""
+function build_executable(
+        library,
+        library_name = splitext(basename(library))[1],
+        cprogram = joinpath(@__DIR__, "..", "examples", "program.c");
+
+        snoopfile = nothing, builddir = "build",
+        verbose = false, quiet = false,
+        cpu_target = nothing, optimize = nothing, debug = nothing,
+        inline = nothing, check_bounds = nothing, math_mode = nothing
+    )
+    if snoopfile != nothing
+        precompfile = joinpath(builddir, "precompiled.jl")
+        snoop(snoopfile, precompfile, joinpath(builddir, "snoop.csv"))
+        jlmain = joinpath(builddir, "julia_main.jl")
+        open(jlmain, "w") do io
+            println(io, "include(\"$(escape_string(precompfile))\")")
+            println(io, "include(\"$(escape_string(library))\")")
+        end
+        library = jlmain
+    end
+    julia_compile(
+
+        library, julia_program_basename = library_name,
+
+        cpu_target = cpu_target, optimize = optimize,
+        debug = debug, inline = inline, check_bounds = check_bounds,
+        math_mode = math_mode, verbose = verbose, quiet = quiet,
+
+        cprog = cprogram, builddir = builddir,
+        clean = false, sysimage = nothing,
+        compile = nothing, depwarn = nothing, autodeps = false,
+        object = true, shared = true, executable = true, julialibs = true,
+    )
+end
+
 
 """
+    build_native_image()
 Builds a clean system image, similar to a fresh Julia install.
 Can also be used to build a native system image for a downloaded cross compiled julia binary.
 """
-function build_native_image(debug = false)
+function build_native_image(debug = false) # debug is ignored right now
     backup = sysimgbackup_folder()
+    # Build in backup dir, so we have a clean backup!
     compile_system_image(joinpath(backup, "sys"), "native")
     copy_system_image(backup, default_sysimg_path(debug))
 end

src/static_julia.jl

@@ -16,6 +16,7 @@ end
 
 system_compiler() = gcc
 bitness_flag() = Int == Int32 ? "-m32" : "-m64"
+executable_ext() = (iswindows() ? ".exe" : "")
 
 function mingw_dir(folders...)
     joinpath(
@@ -78,7 +79,6 @@ function julia_compile(
     julia_program = abspath(julia_program)
     isfile(julia_program) || error("Cannot find file:\n  \"$julia_program\"")
     quiet || println("Julia program file:\n  \"$julia_program\"")
-
     if executable
         cprog = cprog == nothing ? joinpath(@__DIR__, "..", "examples", "program.c") : abspath(cprog)
         isfile(cprog) || error("Cannot find file:\n  \"$cprog\"")
@@ -113,7 +113,7 @@ function julia_compile(
 
     o_file = julia_program_basename * ".o"
     s_file = julia_program_basename * ".$(Libdl.dlext)"
-    e_file = julia_program_basename * (iswindows() ? ".exe" : "")
+    e_file = julia_program_basename * executable_ext()
     tmp_dir = "tmp_v$VERSION"
 
     object && build_object(
@@ -124,7 +124,7 @@ function julia_compile(
 
     shared && build_shared(s_file, joinpath(tmp_dir, o_file), verbose)
 
-    executable && build_executable(s_file, e_file, cprog, verbose)
+    executable && compile_executable(s_file, e_file, cprog, verbose)
 
     julialibs && sync_julia_files(verbose)
 
@@ -151,7 +151,7 @@ function build_shared(s_file, o_file, verbose = false)
     flags = julia_flags()
     command = `$cc $bitness -shared -o $s_file $o_file $flags`
     if isapple()
-        command = `$command -Wl,-install_name,@rpath/\"$s_file\"`
+        command = `$command -Wl,-install_name,@rpath/$s_file`
     elseif iswindows()
         command = `$command -Wl,--export-all-symbols`
     end
@@ -160,7 +160,7 @@ function build_shared(s_file, o_file, verbose = false)
 end
 
 
-function build_executable(s_file, e_file, cprog, verbose = false)
+function compile_executable(s_file, e_file, cprog, verbose = false)
     bitness = bitness_flag()
     cc = system_compiler()
     flags = julia_flags()
@@ -241,16 +241,12 @@ function sync_julia_files(verbose)
         if iswindows() || isapple()
             append!(libfiles, joinpath.(dir, filter(x -> endswith(x, dlext), readdir(dir))))
         else
-            append!(libfiles, joinpath.(dir, filter(x -> contains(x, r"^lib.+\.so(?:\.\d+)*$"), readdir(dir))))
+            append!(libfiles, joinpath.(dir, filter(x -> contains07(x, r"^lib.+\.so(?:\.\d+)*$"), readdir(dir))))
         end
     end
     sync = false
     for src in libfiles
-        if julia_v07
-            contains(src, r"debug") && continue
-        else
-            ismatch(r"debug", src) && continue
-        end
+        contains07(src, r"debug") && continue
         dst = basename(src)
         if filesize(src) != filesize(dst) || ctime(src) > ctime(dst) || mtime(src) > mtime(dst)
             verbose && println("  $dst")

src/system_image.jl

@@ -11,7 +11,9 @@ function compile_system_image(sysimg_path, cpu_target; debug = false)
     # Enter base and setup some useful paths
     base_dir = dirname(Base.find_source_file("sysimg.jl"))
     cd(base_dir) do
-        julia = "julia"
+        # This can probably get merged with build_object.
+        # At some point, I will need to understand build_object a bit better before doing that move, though!
+        julia = Base.julia_cmd().exec[1]
         cc = system_compiler()
         # Ensure we have write-permissions to wherever we're trying to write to
         try
@@ -32,26 +34,6 @@ function compile_system_image(sysimg_path, cpu_target; debug = false)
         info("$julia -C $cpu_target --output-ji $sysimg_path.ji --output-o $sysimg_path.o -J $inference_path.ji --startup-file=no sysimg.jl")
         run(`$julia -C $cpu_target --output-ji $sysimg_path.ji --output-o $sysimg_path.o -J $inference_path.ji --startup-file=no sysimg.jl`)
 
-        link_sysimg(sysimg_path, cc, debug)
+        build_shared("$sysimg_path.$(Libdl.dlext)", "$sysimg_path.o", true)
     end
 end
-
-# Link sys.o into sys.$(dlext)
-function link_sysimg(sysimg_path, cc = system_compiler(), debug = false)
-
-    julia_libdir = dirname(Libdl.dlpath(debug ? "libjulia-debug" : "libjulia"))
-
-    FLAGS = ["-L$julia_libdir"]
-
-    push!(FLAGS, "-shared")
-    push!(FLAGS, debug ? "-ljulia-debug" : "-ljulia")
-    if is_windows()
-        push!(FLAGS, "-lssp")
-    end
-
-    sysimg_file = "$sysimg_path.$(Libdl.dlext)"
-    info("Linking sys.$(Libdl.dlext)")
-    info("$cc $(join(FLAGS, ' ')) -o $sysimg_file $sysimg_path.o")
-    # Windows has difficulties overwriting a file in use so we first link to a temp file
-    run(`$cc $FLAGS -o $sysimg_file $sysimg_path.o`)
-end

test/REQUIRE

@@ -1,2 +1,3 @@
 Matcha
 UnicodeFun
+UnicodePlots