diff --git contrib/generate_precompile.jl contrib/generate_precompile.jl index 6788558275..11e9de09ae 100644 --- contrib/generate_precompile.jl +++ contrib/generate_precompile.jl @@ -8,7 +8,7 @@ if !isdefined(Base, :uv_eventloop) Base.reinit_stdio() end Base.include(@__MODULE__, joinpath(Sys.BINDIR, "..", "share", "julia", "test", "testhelpers", "FakePTYs.jl")) -import .FakePTYs: with_fake_pty +import .FakePTYs: open_fake_pty CTRL_C = '\x03' UP_ARROW = "\e[A" @@ -43,6 +43,12 @@ if Pkg !== nothing precompile_script *= Pkg.precompile_script end +push!(LOAD_PATH, Sys.STDLIB) +using Sockets +Sockets.__init__() +using Libdl +empty!(LOAD_PATH) + function generate_precompile_statements() start_time = time() @@ -62,82 +68,106 @@ function generate_precompile_statements() empty!(DEPOT_PATH) end - # Create a staging area where all the loaded packages are available - PrecompileStagingArea = Module() - for (_pkgid, _mod) in Base.loaded_modules - if !(_pkgid.name in ("Main", "Core", "Base")) - eval(PrecompileStagingArea, :($(Symbol(_mod)) = $_mod)) + print("Generating precompile statements...") + sysimg = Base.unsafe_string(Base.JLOptions().image_file) + mktemp() do precompile_file, _ + # Run a repl process and replay our script + repl_output_buffer = IOBuffer() + @static if Sys.iswindows() + # Fake being cygwin + pipename = """\\\\?\\pipe\\cygwin-$("0"^16)-pty10-abcdef""" + server = listen(pipename) + slave = connect(pipename) + @assert ccall(:jl_ispty, Cint, (Ptr{Cvoid},), slave.handle) == 1 + master = accept(server) + else + slave, master = open_fake_pty() end - end - - # TODO: Implement REPL replayer for Windows - @static if !Sys.iswindows() - print("Generating precompile statements...") - sysimg = isempty(ARGS) ? joinpath(dirname(Sys.BINDIR), "lib", "julia", "sys.ji") : ARGS[1] - - mktemp() do precompile_file, _ - # Run a repl process and replay our script - stdout_accumulator, stderr_accumulator = IOBuffer(), IOBuffer() - with_fake_pty() do slave, master - with_fake_pty() do slave_err, master_err - done = false - withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing, - "TERM" => "") do - p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg - --startup-file=no --color=yes`, - slave, slave, slave_err; wait=false) - readuntil(master, "julia>", keep=true) - for (tty, accumulator) in (master => stdout_accumulator, - master_err => stderr_accumulator) - @async begin - while true - done && break - write(accumulator, readavailable(tty)) - end - end - end - if have_repl - for l in split(precompile_script, '\n'; keepempty=false) - write(master, l, '\n') - end + done = false + withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing, + "TERM" => "") do + if have_repl + p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg + --compile=all --startup-file=no --color=yes`, + slave, slave, slave; wait=false) + readuntil(master, "julia>", keep=true) + t = @async begin + while true + sleep(0.5) + s = String(readavailable(master)) + write(repl_output_buffer, s) + if occursin("__PRECOMPILE_END__", s) + break end - write(master, "exit()\n") - wait(p) - done = true end end + if have_repl + for l in split(precompile_script, '\n'; keepempty=false) + write(master, l, '\n') + end + end + write(master, "print(\"__PRECOMPILE\", \"_END__\")", '\n') + wait(t) + + # TODO Figure out why exit() on Windows doesn't exit the process + if Sys.iswindows() + print(master, "ccall(:_exit, Cvoid, (Cint,), 0)\n") + else + write(master, "exit()\n") + readuntil(master, "exit()\r\e[13C\r\n") + @assert bytesavailable(master) == 0 + end + wait(p) + else + # Is this even needed or is this already recorded just from starting this process? + p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg + --compile=all --startup-file=no + -e0`) end + end + close(master) - # Check what the REPL displayed - # stdout_output = String(take!(stdout_accumulator)) - # println(stdout_output) + # Check what the REPL displayed + # repl_output = String(take!(repl_output_buffer)) + # println(repl_output) - # Extract the precompile statements from stderr - statements = Set{String}() - for statement in split(read(precompile_file, String), '\n') - occursin("Main.", statement) && continue - push!(statements, statement) - end + # Extract the precompile statements from stderr + statements = Set{String}() + for statement in split(read(precompile_file, String), '\n') + occursin("Main.", statement) && continue + push!(statements, statement) + end - # Load the precompile statements - statements_ordered = join(sort(collect(statements)), '\n') - # println(statements_ordered) - if have_repl - # Seems like a reasonable number right now, adjust as needed - @assert length(statements) > 700 + if have_repl + # Seems like a reasonable number right now, adjust as needed + # comment out if debugging script + @assert length(statements) > 700 + end + + # Create a staging area where all the loaded packages are available + PrecompileStagingArea = Module() + for (_pkgid, _mod) in Base.loaded_modules + if !(_pkgid.name in ("Main", "Core", "Base")) + eval(PrecompileStagingArea, :($(Symbol(_mod)) = $_mod)) end + end - Base.include_string(PrecompileStagingArea, statements_ordered) - print(" $(length(statements)) generated in ") - Base.time_print((time() - start_time) * 10^9) - println() + # Execute the collected precompile statements + include_time = @elapsed for statement in sort(collect(statements)) + # println(statement) + try + Base.include_string(PrecompileStagingArea, statement) + catch ex + @error "Failed to precompile $statement" + rethrow(ex) + end end + print(" $(length(statements)) generated in ") + tot_time = time() - start_time + Base.time_print(tot_time * 10^9) + print(" (overhead "); Base.time_print((tot_time - include_time) * 10^9); println(")") end - # Fall back to explicit list on Windows, might as well include them - # for everyone though - Base.include(PrecompileStagingArea, "precompile_explicit.jl") - return end