From 121dbd97a616d95c9f84da6c1cec7842def1ff48 Mon Sep 17 00:00:00 2001 From: marius Date: Tue, 15 Sep 2020 02:51:57 -0700 Subject: [PATCH 1/4] allow building sysimage with custom PyCall --- src/julia/compile.jl | 29 +++++++++++++++-------------- src/julia/sysimage.py | 28 +++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/src/julia/compile.jl b/src/julia/compile.jl index 6e61562a..424fdce2 100644 --- a/src/julia/compile.jl +++ b/src/julia/compile.jl @@ -1,4 +1,4 @@ -compiler_env, script, output, base_sysimage = ARGS +compiler_env, pycall_env, script, output, base_sysimage = ARGS if VERSION < v"0.7-" error("Unsupported Julia version $VERSION") @@ -11,14 +11,15 @@ Pkg.activate(compiler_env) @info "Loading PackageCompiler..." using PackageCompiler -@info "Installing PyCall..." -Pkg.activate(".") -Pkg.add([ - Pkg.PackageSpec( - name = "PyCall", - uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0", - ), -]) +Pkg.activate(pycall_env) +pycall = Pkg.PackageSpec( + name = "PyCall", + uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0", +) +if !(pycall.uuid in keys(Pkg.dependencies())) + @info "Installing PyCall..." + Pkg.add([pycall]) +end if VERSION >= v"1.5-" mktempdir() do dir @@ -27,24 +28,24 @@ if VERSION >= v"1.5-" create_sysimage( Symbol[]; sysimage_path = tmpimg, - project = ".", + project = pycall_env, base_sysimage = isempty(base_sysimage) ? nothing : base_sysimage, ) - @info "Compiling system image..." + @info "Compiling system image with `PyCall` from $pycall_env..." create_sysimage( [:PyCall]; sysimage_path = output, - project = ".", + project = pycall_env, precompile_execution_file = script, base_sysimage = tmpimg, ) end else - @info "Compiling system image..." + @info "Compiling system image with `PyCall` from $pycall_env..." create_sysimage( [:PyCall], sysimage_path = output, - project = ".", + project = pycall_env, precompile_execution_file = script, base_sysimage = isempty(base_sysimage) ? nothing : base_sysimage, ) diff --git a/src/julia/sysimage.py b/src/julia/sysimage.py index a5548c46..cd2c121c 100644 --- a/src/julia/sysimage.py +++ b/src/julia/sysimage.py @@ -79,6 +79,7 @@ def build_sysimage( script=script_path("precompile.jl"), debug=False, compiler_env="", + pycall_env="", base_sysimage=None, ): if debug: @@ -90,15 +91,24 @@ def build_sysimage( julia_py = julia_py_executable() with temporarydirectory(prefix="tmp.pyjulia.sysimage.") as path: - if not compiler_env: + + if compiler_env: + compiler_env = os.path.abspath(compiler_env) + else: compiler_env = os.path.join(path, "compiler_env") # Not using julia-py to install PackageCompiler to reduce # method re-definition warnings: check_call(install_packagecompiler_cmd(julia, compiler_env), cwd=path) + + if pycall_env: + pycall_env = os.path.abspath(".") + else: + pycall_env = "." # Arguments to ./compile.jl script: compile_args = [ compiler_env, + pycall_env, # script -- ./precompile.jl by default os.path.realpath(script), # output -- path to sys.o file @@ -131,10 +141,18 @@ def main(args=None): "--compiler-env", default="", help=""" - Path to a Julia project with PackageCompiler to be used for - system image compilation. Create a temporary environment with - appropriate PackageCompiler by default or when an empty string - is given. + Path to a Julia project with PackageCompiler to be used for system image + compilation. If none is provided, will create a temporary environment + with the latest version of PackageCompiler. + """, + ) + parser.add_argument( + "--pycall-env", + default="", + help=""" + Path to a Julia project with PyCall to be used for system image + compilation. If none is provided, will create a temporary environment + with the latest version of PyCall. """, ) parser.add_argument( From 943381fe0f6e0d15223253e1a1ecd542899a40a5 Mon Sep 17 00:00:00 2001 From: marius Date: Tue, 15 Sep 2020 03:18:28 -0700 Subject: [PATCH 2/4] dont autoinstall in custom pycall-env / fix 1.3 --- src/julia/compile.jl | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/julia/compile.jl b/src/julia/compile.jl index 424fdce2..2008bf45 100644 --- a/src/julia/compile.jl +++ b/src/julia/compile.jl @@ -12,26 +12,27 @@ Pkg.activate(compiler_env) using PackageCompiler Pkg.activate(pycall_env) -pycall = Pkg.PackageSpec( - name = "PyCall", - uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0", -) -if !(pycall.uuid in keys(Pkg.dependencies())) +if pycall_env == "." @info "Installing PyCall..." - Pkg.add([pycall]) + Pkg.add([ + Pkg.PackageSpec( + name = "PyCall", + uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0", + ), + ]) end if VERSION >= v"1.5-" mktempdir() do dir tmpimg = joinpath(dir, basename(output)) - @info "Compiling a temporary system image without `PyCall`..." + @info "Compiling a temporary system image without PyCall..." create_sysimage( Symbol[]; sysimage_path = tmpimg, project = pycall_env, base_sysimage = isempty(base_sysimage) ? nothing : base_sysimage, ) - @info "Compiling system image with `PyCall` from $pycall_env..." + @info "Compiling system image with PyCall from $pycall_env..." create_sysimage( [:PyCall]; sysimage_path = output, @@ -41,7 +42,7 @@ if VERSION >= v"1.5-" ) end else - @info "Compiling system image with `PyCall` from $pycall_env..." + @info "Compiling system image with PyCall from $pycall_env..." create_sysimage( [:PyCall], sysimage_path = output, From ea9c2a7af48c7026d33eba3961ab2449dfe1a3d1 Mon Sep 17 00:00:00 2001 From: marius Date: Tue, 15 Sep 2020 03:50:06 -0700 Subject: [PATCH 3/4] fix style --- src/julia/sysimage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/julia/sysimage.py b/src/julia/sysimage.py index cd2c121c..a0d2d60f 100644 --- a/src/julia/sysimage.py +++ b/src/julia/sysimage.py @@ -91,7 +91,7 @@ def build_sysimage( julia_py = julia_py_executable() with temporarydirectory(prefix="tmp.pyjulia.sysimage.") as path: - + if compiler_env: compiler_env = os.path.abspath(compiler_env) else: @@ -99,7 +99,7 @@ def build_sysimage( # Not using julia-py to install PackageCompiler to reduce # method re-definition warnings: check_call(install_packagecompiler_cmd(julia, compiler_env), cwd=path) - + if pycall_env: pycall_env = os.path.abspath(".") else: From 302c3148a2abbcf7ee17608c80a82082c88867a7 Mon Sep 17 00:00:00 2001 From: marius Date: Tue, 15 Sep 2020 03:51:13 -0700 Subject: [PATCH 4/4] fix typo --- src/julia/sysimage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/julia/sysimage.py b/src/julia/sysimage.py index a0d2d60f..5cf32749 100644 --- a/src/julia/sysimage.py +++ b/src/julia/sysimage.py @@ -101,7 +101,7 @@ def build_sysimage( check_call(install_packagecompiler_cmd(julia, compiler_env), cwd=path) if pycall_env: - pycall_env = os.path.abspath(".") + pycall_env = os.path.abspath(pycall_env) else: pycall_env = "."