diff --git a/README.md b/README.md index b684f07..8e135d1 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ These functions are intended to be used interactively when the Pkg REPL is not a - `rm(pkg)` removes a dependency or a vector of dependencies. - `add_channel(channel)` adds a channel. - `rm_channel(channel)` removes a channel. -- `add_pip(pkg; version="")` adds/replaces a pip dependency. +- `add_pip(pkg; version="", binary="", editable=false)` adds/replaces a pip dependency. - `rm_pip(pkg)` removes a pip dependency. ### CondaPkg.toml @@ -90,6 +90,7 @@ some-local-package = "@ ./foo.zip" version = "~=2.1" extras = ["email", "timezone"] binary = "no" # or "only" +editable = true # install package with `--editable`/`-e` `pip` argument if `true`: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs ``` ## Access the Conda environment diff --git a/src/PkgREPL.jl b/src/PkgREPL.jl index 1d2099a..89f10ad 100644 --- a/src/PkgREPL.jl +++ b/src/PkgREPL.jl @@ -26,7 +26,7 @@ $ CondaPkg.PkgSpec(name, version = version, channel = channel, build = build) end -function parse_pip_pkg(x::String; binary::String = "") +function parse_pip_pkg(x::String; binary::String = "", editable=false) m = match( r""" ^ @@ -41,7 +41,7 @@ $ name = m.captures[1] extras = split(something(m.captures[3], ""), ",", keepempty = false) version = something(m.captures[4], "") - CondaPkg.PipPkgSpec(name, version = version, binary = binary, extras = extras) + CondaPkg.PipPkgSpec(name, version = version, binary = binary, extras = extras, editable = editable) end function parse_channel(x::String) @@ -62,6 +62,13 @@ const binary_opt = Pkg.REPLMode.OptionDeclaration([ :api => :binary => identity, ]) +const editable_opt = Pkg.REPLMode.OptionDeclaration([ + :name => "editable", + :short_name => "e", + :takes_arg => false, + :api => :editable => true, +]) + ### status function status() @@ -191,13 +198,13 @@ const channel_add_spec = Pkg.REPLMode.CommandSpec( ### pip_add -function pip_add(args; binary = "") - CondaPkg.add([parse_pip_pkg(arg, binary = binary) for arg in args]) +function pip_add(args; binary = "", editable = false) + CondaPkg.add([parse_pip_pkg(arg, binary = binary, editable = editable) for arg in args]) end const pip_add_help = Markdown.parse(""" ``` -conda pip_add [--binary={only|no}] pkg ... +conda pip_add [--binary={only|no}] [--editable] pkg ... ``` Add Pip packages to the environment. @@ -209,6 +216,7 @@ pkg> conda pip_add build pkg> conda pip_add build~=0.7 # version range pkg> conda pip_add pydantic[email,timezone] # extras pkg> conda pip_add --binary=no nmslib # always build from source +pkg> conda pip_add --editable nmslib@/path/to/package_source # install locally ``` """) @@ -219,7 +227,7 @@ const pip_add_spec = Pkg.REPLMode.CommandSpec( help = pip_add_help, description = "add Pip packages", arg_count = 0 => Inf, - option_spec = [binary_opt], + option_spec = [binary_opt, editable_opt], ) ### rm diff --git a/src/deps.jl b/src/deps.jl index a41fa99..ad12478 100644 --- a/src/deps.jl +++ b/src/deps.jl @@ -80,6 +80,7 @@ function parse_deps(toml) version = "" binary = "" extras = String[] + editable = false if dep isa AbstractString version = _convert(String, dep) elseif dep isa AbstractDict @@ -90,16 +91,18 @@ function parse_deps(toml) binary = _convert(String, v) elseif k == "extras" extras = _convert(Vector{String}, v) + elseif k == "editable" + editable = _convert(Bool, v) else error( - "pip.deps keys must be 'version', 'extras' or 'binary', got '$k'", + "pip.deps keys must be 'version', 'extras', 'binary' or 'editable', got '$k'", ) end end else error("pip.deps must be String or Dict, got $(typeof(dep))") end - pkg = PipPkgSpec(name, version = version, binary = binary, extras = extras) + pkg = PipPkgSpec(name, version = version, binary = binary, extras = extras, editable = editable) push!(pip_packages, pkg) end end @@ -231,6 +234,9 @@ function status(; io::IO = stderr) if !isempty(pkg.extras) push!(specparts, "[$(join(pkg.extras, ", "))]") end + if pkg.editable + push!(specparts, "editable") + end isempty(specparts) || printstyled(io, " (", join(specparts, ", "), ")", color = :light_black) println(io) @@ -340,6 +346,9 @@ function add!(toml, pkg::PipPkgSpec) if !isempty(pkg.extras) dep["extras"] = pkg.extras end + if pkg.editable + dep["editable"] = pkg.editable + end if issubset(keys(dep), ["version"]) deps[pkg.name] = pkg.version else @@ -405,7 +414,7 @@ Removes a channel from the current environment. rm_channel(channel::AbstractString; kw...) = rm(ChannelSpec(channel); kw...) """ - add_pip(pkg; version="", binary="", extras=[], resolve=true) + add_pip(pkg; version="", binary="", extras=[], resolve=true, editable=false) Adds a pip dependency to the current environment. @@ -414,8 +423,8 @@ Adds a pip dependency to the current environment. Use conda dependencies instead if at all possible. Pip does not handle version conflicts gracefully, so it is possible to get incompatible versions. """ -add_pip(pkg::AbstractString; version = "", binary = "", extras = String[], kw...) = - add(PipPkgSpec(pkg, version = version, binary = binary, extras = extras); kw...) +add_pip(pkg::AbstractString; version = "", binary = "", extras = String[], editable = false, kw...) = + add(PipPkgSpec(pkg, version = version, binary = binary, extras = extras, editable = editable); kw...) """ rm_pip(pkg; resolve=true) diff --git a/src/meta.jl b/src/meta.jl index 5902169..f0dfe2f 100644 --- a/src/meta.jl +++ b/src/meta.jl @@ -4,7 +4,7 @@ information about the most recent resolve. """ # increment whenever the metadata format changes -const META_VERSION = 14 +const META_VERSION = 15 @kwdef mutable struct Meta timestamp::Float64 @@ -48,6 +48,9 @@ end function read_meta(io::IO, ::Type{Symbol}) Symbol(read_meta(io, String)) end +function read_meta(io::IO, ::Type{Bool}) + read(io, Bool) +end function read_meta(io::IO, ::Type{Vector{T}}) where {T} len = read(io, Int) ans = Vector{T}() @@ -76,7 +79,8 @@ function read_meta(io::IO, ::Type{PipPkgSpec}) version = read_meta(io, String) binary = read_meta(io, String) extras = read_meta(io, Vector{String}) - PipPkgSpec(name, version = version, binary = binary, extras = extras) + editable = read_meta(io, Bool) + PipPkgSpec(name, version = version, binary = binary, extras = extras, editable = editable) end function write_meta(io::IO, meta::Meta) @@ -102,6 +106,9 @@ end function write_meta(io::IO, x::Symbol) write_meta(io, String(x)) end +function write_meta(io::IO, x::Bool) + write(io, x) +end function write_meta(io::IO, x::Vector) write(io, convert(Int, length(x))) for item in x @@ -125,4 +132,5 @@ function write_meta(io::IO, x::PipPkgSpec) write_meta(io, x.version) write_meta(io, x.binary) write_meta(io, x.extras) + write_meta(io, x.editable) end diff --git a/src/resolve.jl b/src/resolve.jl index e0a98f7..2cfe03a 100644 --- a/src/resolve.jl +++ b/src/resolve.jl @@ -464,6 +464,7 @@ function _resolve_merge_pip_packages(packages) urls = String[] binary = "" extras = String[] + editables = Bool[] for (fn, pkg) in pkgs @assert pkg.name == name if startswith(pkg.version, "@") @@ -485,6 +486,7 @@ function _resolve_merge_pip_packages(packages) end end append!(extras, pkg.extras) + push!(editables, pkg.editable) end sort!(unique!(urls)) sort!(unique!(versions)) @@ -500,7 +502,12 @@ function _resolve_merge_pip_packages(packages) "direct references ('@ ...') and version specifiers both given for pip package '$name'", ) end - push!(specs, PipPkgSpec(name, version = version, binary = binary, extras = extras)) + unique!(editables) + if length(editables) != 1 + error("both 'editable = true' and 'editable = false' specified for pip package '$name'") + end + editable = only(editables) + push!(specs, PipPkgSpec(name, version = version, binary = binary, extras = extras, editable = editable)) end sort!(specs, by = x -> x.name) end @@ -615,9 +622,16 @@ function _resolve_pip_install(io, pip_specs, load_path, backend) elseif spec.binary == "no" push!(args, "--no-binary", spec.name) end + if spec.editable + # remove the @ from the beginning of the path. + url = replace(spec.version, r"@\s*"=>"") + push!(args, "--editable", url) + end end for spec in pip_specs - push!(args, specstr(spec)) + if !spec.editable + push!(args, specstr(spec)) + end end vrb = _verbosity_flags() flags = vrb diff --git a/src/spec.jl b/src/spec.jl index 216793d..b0ed120 100644 --- a/src/spec.jl +++ b/src/spec.jl @@ -133,12 +133,14 @@ struct PipPkgSpec version::String binary::String extras::Vector{String} - function PipPkgSpec(name; version = "", binary = "", extras = String[]) + editable::Bool + function PipPkgSpec(name; version = "", binary = "", extras = String[], editable = false) name = validate_pip_pkg(name) version = validate_pip_version(version) binary = validate_pip_binary(binary) extras = validate_pip_extras(extras) - new(name, version, binary, extras) + validate_pip_editable(editable, version) + new(name, version, binary, extras, editable) end end @@ -146,9 +148,10 @@ Base.:(==)(x::PipPkgSpec, y::PipPkgSpec) = (x.name == y.name) && (x.version == y.version) && (x.binary == y.binary) && - (x.extras == y.extras) + (x.extras == y.extras) && + (x.editable == y.editable) Base.hash(x::PipPkgSpec, h::UInt) = - hash(x.extras, hash(x.binary, hash(x.version, hash(x.name, h)))) + hash(x.editable, hash(x.extras, hash(x.binary, hash(x.version, hash(x.name, h))))) is_valid_pip_pkg(name) = occursin(r"^\s*[-_.A-Za-z0-9]+\s*$", name) @@ -225,8 +228,24 @@ function pixispec(x::PipPkgSpec) else spec["git"] = url end + elseif startswith(url, r"[a-z]+://") + # Handle the file:// prefix. + m = match(r"^file://(.*)", url) + if m !== nothing + spec["path"] = m.captures[1] + if x.editable + spec["editable"] = true + end + else + spec["url"] = url + end else - spec["url"] = url + # https://pixi.sh/latest/reference/pixi_manifest/#path + # minimal-project = { path = "./minimal-project", editable = true} + spec["path"] = url + if x.editable + spec["editable"] = true + end end else spec["version"] = x.version == "" ? "*" : x.version @@ -240,3 +259,7 @@ function pixispec(x::PipPkgSpec) spec end end +validate_pip_editable(editable, version) = + if editable && !startswith(version, "@") + error("invalid pip version for editable install: must start with `@` but version is $(version)") + end diff --git a/test/Foo/.gitignore b/test/Foo/.gitignore new file mode 100644 index 0000000..3a0bcd5 --- /dev/null +++ b/test/Foo/.gitignore @@ -0,0 +1,3 @@ +*.egg-info +__pycache__ +build diff --git a/test/Foo/foo/__init__.py b/test/Foo/foo/__init__.py new file mode 100644 index 0000000..2d76aba --- /dev/null +++ b/test/Foo/foo/__init__.py @@ -0,0 +1 @@ +x = 42 diff --git a/test/Foo/foo/test/added.py b/test/Foo/foo/test/added.py new file mode 100644 index 0000000..013dd7b --- /dev/null +++ b/test/Foo/foo/test/added.py @@ -0,0 +1 @@ +y = 3 diff --git a/test/Foo/setup.py b/test/Foo/setup.py new file mode 100644 index 0000000..1fd66ee --- /dev/null +++ b/test/Foo/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup(name="foo") diff --git a/test/FooNonEditable/.gitignore b/test/FooNonEditable/.gitignore new file mode 100644 index 0000000..3a0bcd5 --- /dev/null +++ b/test/FooNonEditable/.gitignore @@ -0,0 +1,3 @@ +*.egg-info +__pycache__ +build diff --git a/test/FooNonEditable/foononeditable/__init__.py b/test/FooNonEditable/foononeditable/__init__.py new file mode 100644 index 0000000..0d51d7c --- /dev/null +++ b/test/FooNonEditable/foononeditable/__init__.py @@ -0,0 +1 @@ +x = -42 diff --git a/test/FooNonEditable/setup.py b/test/FooNonEditable/setup.py new file mode 100644 index 0000000..662d917 --- /dev/null +++ b/test/FooNonEditable/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup(name="foononeditable") diff --git a/test/internals.jl b/test/internals.jl index 5c47cac..a796323 100644 --- a/test/internals.jl +++ b/test/internals.jl @@ -25,6 +25,8 @@ end @test_throws Exception CondaPkg.PipPkgSpec("") @test_throws Exception CondaPkg.PipPkgSpec("foo!") @test_throws Exception CondaPkg.PipPkgSpec("foo", version = "1.2") + @test_throws Exception CondaPkg.PipPkgSpec("foo", editable = true) + @test_throws Exception CondaPkg.PipPkgSpec("foo", version = "1.2", editable = true) spec = CondaPkg.PipPkgSpec(" F...OO_-0 ", version = " @./SOME/Path ") @test spec.name == "f-oo-0" @test spec.version == "@./SOME/Path" @@ -36,6 +38,8 @@ end CondaPkg.PkgSpec("foo", version = "=1.2.3", channel = "bar"), CondaPkg.ChannelSpec("fooo"), CondaPkg.PipPkgSpec("foooo", version = "==2.3.4"), + CondaPkg.PipPkgSpec("bar", version = "@ /abs/path/somewhere", editable = true), + CondaPkg.PipPkgSpec("baz", version = "@ ./rel/path/somewhere", editable = true) ] for spec in specs io = IOBuffer() diff --git a/test/main.jl b/test/main.jl index 8f45a79..3f7dae8 100644 --- a/test/main.jl +++ b/test/main.jl @@ -163,20 +163,75 @@ end end end -@testitem "install/remove executable package" begin +@testitem "pip install/remove a local python package" begin include("setup.jl") - if !isnull - CondaPkg.add("uv", resolve = false) - CondaPkg.resolve(force = true) - exe_path = CondaPkg.which("uv") - @test exe_path !== nothing - @test isfile(exe_path) - CondaPkg.rm("uv", resolve = false) - CondaPkg.resolve(force = true) - if !ispixi - # pixi doesn't seem to remove unused packages?? - @test !isfile(exe_path) - end + CondaPkg.add("python", version="==3.10.2") + # verify package isn't already installed + @test !occursin("foo", status()) + CondaPkg.withenv() do + isnull || @test_throws Exception run(`python -c "import foo"`) + end + + # install package + # The directory with the setup.py file (here `Foo`) needs to be different from the name of the Python module (here `foo`), otherwise `import foo` will never throw an exception and the tests checking that the package isn't installed will fail. + pkg_path = joinpath(dirname(@__FILE__), "FooNonEditable") + CondaPkg.add_pip("foononeditable", version="@ $(pkg_path)") + @test occursin("foononeditable", status()) + @test occursin(pkg_path, status()) + CondaPkg.withenv() do + isnull || run(`python -c "import foononeditable"`) + end + + # remove package + CondaPkg.rm_pip("foononeditable") + @test !occursin("foononeditable", status()) + CondaPkg.withenv() do + isnull || @test_throws Exception run(`python -c "import foononeditable"`) + end +end + +@testitem "pip install/remove a local editable python package" begin + include("setup.jl") + CondaPkg.add("python", version="==3.10.2") + # verify package isn't already installed + @test !occursin("foo", status()) + CondaPkg.withenv() do + isnull || @test_throws Exception run(`python -c "import foo"`) + end + + # install package + # The directory with the setup.py file (here `Foo`) needs to be different from the name of the Python module (here `foo`), otherwise `import foo` will never throw an exception and the tests checking that the package isn't installed will fail. + pkg_path = joinpath(dirname(@__FILE__), "Foo") + CondaPkg.add_pip("foo", version="@ $(pkg_path)", editable=true) + @test occursin("foo", status()) + @test occursin(pkg_path, status()) + CondaPkg.withenv() do + isnull || run(`python -c "import foo"`) + end + + # The `added` module shouldn't exist. + CondaPkg.withenv() do + isnull || @test_throws Exception run(`python -c "import foo.added"`) + end + + # Now add the `added.py` file to create the `added` module. + added_src_path = joinpath(dirname(@__FILE__), "Foo", "foo", "test", "added.py") + added_dst_path = joinpath(dirname(@__FILE__), "Foo", "foo", "added.py") + cp(added_src_path, added_dst_path) + + # Test that the `added` module exists. + CondaPkg.withenv() do + isnull || run(`python -c "import foo.added; print(foo.added.y)"`) + end + + # Remove the added file for later tests. + rm(added_dst_path) + + # remove package + CondaPkg.rm_pip("foo") + @test !occursin("foo", status()) + CondaPkg.withenv() do + isnull || @test_throws Exception run(`python -c "import foo"`) end end