Skip to content

Commit 266eaf1

Browse files
authored
add incremental compilation + improve snooping
* fix snooping * fix precompile file * fixes to compile more * rename * fix remaining issues * do the snoopy thingy * add cleaner incremental solution * escape correctly * don't rebuild for snooping * fix typo * fix broadcast * work with non toml packages * clean up snoop + incremental - fix bugs * fix some more bugs: * add docstrings * add the packages we need to manifest * add tests * better tests + fixes for tests * remove failing tests * remove more * fix tests * fix tests * correctly include pgk compiler * fix windows bugs * no success * Update src/incremental.jl Co-Authored-By: SimonDanisch <sdanisch@gmail.com> * Apply suggestions from code review Co-Authored-By: SimonDanisch <sdanisch@gmail.com> * cosmetics * correctly implement run_julia * fix flags * fix tags + add multi package compile * use correct flags * use package toml * parse test REQUIRE even if toml is present * allow snoop file to error * fix atexit * use sysimgfolder idiomatically * also add packages from toml * fix bug + easier reuse
1 parent 854206e commit 266eaf1

File tree

7 files changed

+695
-110
lines changed

7 files changed

+695
-110
lines changed

src/PackageCompiler.jl

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
module PackageCompiler
22

33
using Libdl, SnoopCompile
4-
Sys.iswindows() && using WinRPM
54

65
include("compiler_flags.jl")
76
include("static_julia.jl")
87
include("api.jl")
98
include("snooping.jl")
109
include("system_image.jl")
10+
include("pkg.jl")
11+
include("incremental.jl")
12+
1113

1214
const sysimage_binaries = ("sys.$(Libdl.dlext)",)
1315

@@ -93,16 +95,27 @@ function compile_package(packages...; kw_args...)
9395
end
9496

9597
"""
96-
compile_package(packages::Tuple{String, String}...; force = false, reuse = false, debug = false, cpu_target = nothing)
98+
compile_package(
99+
packages::Tuple{String, String}...;
100+
force = false, reuse = false, debug = false, cpu_target = nothing,
101+
additional_packages = Symbol[]
102+
)
97103
98104
Compile a list of packages. Each package comes as a tuple of `(package_name, precompile_file)`
99105
where the precompile file should contain all function calls, that should get compiled into the system image.
100106
Usually the `runtests.jl` file is a good candidate, since it should run all important functions of a package.
107+
You can pass `additional_packages` a vector of symbols with package names, to help AOT compiling
108+
uninstalled, recursive dependencies of `packages`. Look at `compile_incremental` to
109+
use a toml instead.
101110
"""
102-
function compile_package(packages::Tuple{String, String}...; force = false, reuse = false, debug = false, cpu_target = nothing)
111+
function compile_package(
112+
packages::Tuple{String, String}...;
113+
force = false, reuse = false, debug = false, cpu_target = nothing,
114+
additional_packages = Symbol[]
115+
)
103116
userimg = sysimg_folder("precompile.jl")
104117
if !reuse
105-
snoop_userimg(userimg, packages...)
118+
snoop_userimg(userimg, packages...; additional_packages = additional_packages)
106119
end
107120
!isfile(userimg) && reuse && error("Nothing to reuse. Please run `compile_package(reuse = true)`")
108121
image_path = sysimg_folder()

src/compiler_flags.jl

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,3 +61,200 @@ end
6161
function allflags()
6262
return "$(cflags()) $(ldflags()) $(ldlibs())"
6363
end
64+
65+
66+
67+
68+
const short_flag_to_jloptions = Dict(
69+
"C" => :cpu_target,
70+
"J" => :image_file,
71+
"O" => :opt_level,
72+
"g" => :debug_level
73+
)
74+
75+
const flag_to_jloptions = Dict(
76+
"check-bounds" => :check_bounds,
77+
"code-coverage" => :code_coverage,
78+
"compile" => :compile_enabled,
79+
"compiled-modules" => :use_compiled_modules,
80+
"cpu-target" => :cpu_target,
81+
"depwarn" => :depwarn,
82+
"handle-signals" => :handle_signals,
83+
"history-file" => :historyfile,
84+
"machine-file" => :machine_file,
85+
"math-mode" => :fast_math,
86+
"optimize" => :opt_level,
87+
"output-bc" => :outputbc,
88+
"output-ji" => :outputji,
89+
"output-jitbc" => :outputjitbc,
90+
"output-o" => :outputo,
91+
"output-unoptbc" => :outputunoptbc,
92+
"project" => :project,
93+
"startup-file" => :startupfile,
94+
"sysimage" => :image_file,
95+
"sysimage-native-code" => :use_sysimage_native_code,
96+
"trace-compile" => :trace_compile,
97+
"track-allocation" => :malloc_log,
98+
"warn-overwrite" => :warn_overwrite,
99+
"inline" => :can_inline
100+
)
101+
102+
const jl_options_to_flag = Dict(
103+
:can_inline => "inline",
104+
:handle_signals => "handle-signals",
105+
:opt_level => "optimize",
106+
:depwarn => "depwarn",
107+
:malloc_log => "track-allocation",
108+
:outputo => "output-o",
109+
:startupfile => "startup-file",
110+
:compile_enabled => "compile",
111+
:trace_compile => "trace-compile",
112+
:check_bounds => "check-bounds",
113+
:outputji => "output-ji",
114+
:use_sysimage_native_code => "sysimage-native-code",
115+
:outputunoptbc => "output-unoptbc",
116+
:historyfile => "history-file",
117+
:outputbc => "output-bc",
118+
:warn_overwrite => "warn-overwrite",
119+
:machine_file => "machine-file",
120+
:code_coverage => "code-coverage",
121+
:image_file => "sysimage",
122+
:cpu_target => "cpu-target",
123+
:outputjitbc => "output-jitbc",
124+
:project => "project",
125+
:fast_math => "math-mode",
126+
:use_compiled_modules => "compiled-modules",
127+
:debug_level => "g"
128+
)
129+
130+
const flags_with_cmdval = Set([
131+
:handle_signals,
132+
:use_sysimage_native_code,
133+
:depwarn,
134+
:can_inline,
135+
:historyfile,
136+
:startupfile,
137+
:use_compiled_modules,
138+
:warn_overwrite,
139+
:check_bounds,
140+
:fast_math,
141+
:compile_enabled,
142+
:malloc_log,
143+
:code_coverage
144+
])
145+
146+
function to_cmd_val(key::Symbol, val)
147+
# undocumented auto!? well we can only skip it I guess
148+
if key in (:depwarn, :warn_overwrite)
149+
val == 0 && return "no"
150+
val == 1 && return "yes"
151+
val == 2 && return "error"
152+
end
153+
if key in (:code_coverage, :malloc_log)
154+
val == 0 && return "none"
155+
val == 1 && return "user"
156+
val == 2 && return "all"
157+
end
158+
if key == :compile_enabled
159+
val == 0 && return "no"
160+
val == 1 && return "yes"
161+
val == 2 && return "all"
162+
end
163+
if key == :fast_math
164+
val == 0 && return "ieee"
165+
val == 1 && return "fast"
166+
end
167+
val in (0, -1) && return ""
168+
val == 1 && return "yes"
169+
val == 2 && return "no"
170+
end
171+
172+
173+
function jl_option_value(opts, key)
174+
value = getfield(opts, key)
175+
if value isa Ptr{UInt8}
176+
return value == C_NULL ? "" : unsafe_string(value)
177+
end
178+
if key in flags_with_cmdval
179+
return to_cmd_val(key, value)
180+
end
181+
return value
182+
end
183+
184+
is_short_flag(flag) = haskey(short_flag_to_jloptions, flag)
185+
186+
function jl_option_key(flag::Symbol)
187+
# if symbol is used, we can also check the fields directly.
188+
# TODO should we also do this for strings?
189+
flag in fieldnames(Base.JLOptions) && return flag
190+
jl_option_key(string(flag))
191+
end
192+
193+
function jl_option_key(flag::String)
194+
haskey(short_flag_to_jloptions, flag) && return short_flag_to_jloptions[flag]
195+
haskey(flag_to_jloptions, flag) && return flag_to_jloptions[flag]
196+
m_flag = replace(flag, "_" => "-")
197+
haskey(flag_to_jloptions, m_flag) && return flag_to_jloptions[m_flag]
198+
flags = replace.([keys(flag_to_jloptions)..., keys(short_flag_to_jloptions)...], ("-" => "_",))
199+
error("Flag $flag not a valid Julia Options. Valid flags are:\n$(join(flags, " "))")
200+
end
201+
202+
"""
203+
extract_flag(flag, jl_cmd = Base.julia_cmd())
204+
205+
Extracts the value for `flag` from `jl_cmd`.
206+
"""
207+
function jl_flag_value(flag, jl_options = Base.JLOptions())
208+
jl_option_value(jl_options, jl_option_key(flag))
209+
end
210+
211+
current_systemimage() = jl_flag_value("J")
212+
213+
"""
214+
run_julia(
215+
code::String;
216+
g = nothing, O = nothing, output_o = nothing, J = nothing,
217+
startup_file = "no"
218+
)
219+
220+
Run `code` in a julia command.
221+
You can overwrite any julia command line flag by setting it to a value.
222+
If the flag has the value `nothing`, the value of the flag of the current julia process is used.
223+
"""
224+
function run_julia(code::String; kw...)
225+
run(julia_code_cmd(code; kw...))
226+
end
227+
228+
function jl_command(flag, value)
229+
isempty(value) && return ""
230+
if is_short_flag(flag)
231+
string("-", flag, value)
232+
else
233+
string("--", flag, "=", value)
234+
end
235+
end
236+
237+
function julia_code_cmd(
238+
code::String, jl_options = Base.JLOptions();
239+
kw...
240+
)
241+
jl_cmd = Base.julia_cmd()
242+
cmd = `$(jl_cmd.exec[1])` # extract julia exe
243+
# Add the commands from the keyword arguments
244+
for (k, v) in kw
245+
jl_key = jl_option_key(k)
246+
flag = jl_options_to_flag[jl_key]
247+
push!(cmd.exec, jl_command(flag, v))
248+
end
249+
# add remaining commands from JLOptions
250+
for key in setdiff(keys(jl_options_to_flag), keys(kw))
251+
flag = jl_options_to_flag[key]
252+
command = jl_command(flag, jl_option_value(jl_options, key))
253+
isempty(command) || push!(cmd.exec, command)
254+
end
255+
# for better debug, let's not make a tmp file which would get lost!
256+
file = sysimg_folder("run_julia_code.jl")
257+
open(io-> println(io, code), file, "w")
258+
push!(cmd.exec, file)
259+
cmd
260+
end

0 commit comments

Comments
 (0)