Add pip install --editable functionality

README.md

@@ -57,7 +57,7 @@ These functions are intended to be used interactively when the Pkg REPL is not a
 - `rm(pkg)` removes a dependency or a vector of dependencies.
 - `add_channel(channel)` adds a channel.
 - `rm_channel(channel)` removes a channel.
-- `add_pip(pkg; version="")` adds/replaces a pip dependency.
+- `add_pip(pkg; version="", binary="", editable=false)` adds/replaces a pip dependency.
 - `rm_pip(pkg)` removes a pip dependency.
 
 ### CondaPkg.toml
@@ -90,6 +90,7 @@ some-local-package = "@ ./foo.zip"
 version = "~=2.1"
 extras = ["email", "timezone"]
 binary = "no"  # or "only"
+editable = true  # install package with `--editable`/`-e` `pip` argument if `true`: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs
 ```
 
 ## Access the Conda environment

src/PkgREPL.jl

@@ -26,7 +26,7 @@ $
     CondaPkg.PkgSpec(name, version = version, channel = channel, build = build)
 end
 
-function parse_pip_pkg(x::String; binary::String = "")
+function parse_pip_pkg(x::String; binary::String = "", editable=false)
     m = match(
         r"""
 ^
@@ -41,7 +41,7 @@ $
     name = m.captures[1]
     extras = split(something(m.captures[3], ""), ",", keepempty = false)
     version = something(m.captures[4], "")
-    CondaPkg.PipPkgSpec(name, version = version, binary = binary, extras = extras)
+    CondaPkg.PipPkgSpec(name, version = version, binary = binary, extras = extras, editable = editable)
 end
 
 function parse_channel(x::String)
@@ -62,6 +62,13 @@ const binary_opt = Pkg.REPLMode.OptionDeclaration([
     :api => :binary => identity,
 ])
 
+const editable_opt = Pkg.REPLMode.OptionDeclaration([
+    :name => "editable",
+    :short_name => "e",
+    :takes_arg => false,
+    :api => :editable => true,
+])
+
 ### status
 
 function status()
@@ -191,13 +198,13 @@ const channel_add_spec = Pkg.REPLMode.CommandSpec(
 
 ### pip_add
 
-function pip_add(args; binary = "")
-    CondaPkg.add([parse_pip_pkg(arg, binary = binary) for arg in args])
+function pip_add(args; binary = "", editable = false)
+    CondaPkg.add([parse_pip_pkg(arg, binary = binary, editable = editable) for arg in args])
 end
 
 const pip_add_help = Markdown.parse("""
 ```
-conda pip_add [--binary={only|no}] pkg ...
+conda pip_add [--binary={only|no}] [--editable] pkg ...
 ```
 
 Add Pip packages to the environment.
@@ -209,6 +216,7 @@ pkg> conda pip_add build
 pkg> conda pip_add build~=0.7                # version range
 pkg> conda pip_add pydantic[email,timezone]  # extras
 pkg> conda pip_add --binary=no nmslib        # always build from source
+pkg> conda pip_add --editable nmslib@/path/to/package_source  # install locally
 ```
 """)
 
@@ -219,7 +227,7 @@ const pip_add_spec = Pkg.REPLMode.CommandSpec(
     help = pip_add_help,
     description = "add Pip packages",
     arg_count = 0 => Inf,
-    option_spec = [binary_opt],
+    option_spec = [binary_opt, editable_opt],
 )
 
 ### rm

src/deps.jl

@@ -80,6 +80,7 @@ function parse_deps(toml)
                 version = ""
                 binary = ""
                 extras = String[]
+                editable = false
                 if dep isa AbstractString
                     version = _convert(String, dep)
                 elseif dep isa AbstractDict
@@ -90,16 +91,18 @@ function parse_deps(toml)
                             binary = _convert(String, v)
                         elseif k == "extras"
                             extras = _convert(Vector{String}, v)
+                        elseif k == "editable"
+                            editable = _convert(Bool, v)
                         else
                             error(
-                                "pip.deps keys must be 'version', 'extras' or 'binary', got '$k'",
+                                "pip.deps keys must be 'version', 'extras', 'binary' or 'editable', got '$k'",
                             )
                         end
                     end
                 else
                     error("pip.deps must be String or Dict, got $(typeof(dep))")
                 end
-                pkg = PipPkgSpec(name, version = version, binary = binary, extras = extras)
+                pkg = PipPkgSpec(name, version = version, binary = binary, extras = extras, editable = editable)
                 push!(pip_packages, pkg)
             end
         end
@@ -231,6 +234,9 @@ function status(; io::IO = stderr)
             if !isempty(pkg.extras)
                 push!(specparts, "[$(join(pkg.extras, ", "))]")
             end
+            if pkg.editable
+                push!(specparts, "editable")
+            end
             isempty(specparts) ||
                 printstyled(io, " (", join(specparts, ", "), ")", color = :light_black)
             println(io)
@@ -302,6 +308,9 @@ function add!(toml, pkg::PipPkgSpec)
     if !isempty(pkg.extras)
         dep["extras"] = pkg.extras
     end
+    if pkg.editable
+        dep["editable"] = pkg.editable
+    end
     if issubset(keys(dep), ["version"])
         deps[pkg.name] = pkg.version
     else
@@ -380,7 +389,7 @@ Removes a channel from the current environment.
 rm_channel(channel::AbstractString; kw...) = rm(ChannelSpec(channel); kw...)
 
 """
-    add_pip(pkg; version="", binary="", extras=[], resolve=true)
+    add_pip(pkg; version="", binary="", extras=[], resolve=true, editable=false)
 
 Adds a pip dependency to the current environment.
 
@@ -389,8 +398,8 @@ Adds a pip dependency to the current environment.
     Use conda dependencies instead if at all possible. Pip does not handle version
     conflicts gracefully, so it is possible to get incompatible versions.
 """
-add_pip(pkg::AbstractString; version = "", binary = "", extras = String[], kw...) =
-    add(PipPkgSpec(pkg, version = version, binary = binary, extras = extras); kw...)
+add_pip(pkg::AbstractString; version = "", binary = "", extras = String[], editable = false, kw...) =
+    add(PipPkgSpec(pkg, version = version, binary = binary, extras = extras, editable = editable); kw...)
 
 """
     rm_pip(pkg; resolve=true)

src/meta.jl

@@ -4,7 +4,7 @@ information about the most recent resolve.
 """
 
 # increment whenever the metadata format changes
-const META_VERSION = 14
+const META_VERSION = 15
 
 @kwdef mutable struct Meta
     timestamp::Float64
@@ -48,6 +48,9 @@ end
 function read_meta(io::IO, ::Type{Symbol})
     Symbol(read_meta(io, String))
 end
+function read_meta(io::IO, ::Type{Bool})
+    read(io, Bool)
+end
 function read_meta(io::IO, ::Type{Vector{T}}) where {T}
     len = read(io, Int)
     ans = Vector{T}()
@@ -76,7 +79,8 @@ function read_meta(io::IO, ::Type{PipPkgSpec})
     version = read_meta(io, String)
     binary = read_meta(io, String)
     extras = read_meta(io, Vector{String})
-    PipPkgSpec(name, version = version, binary = binary, extras = extras)
+    editable = read_meta(io, Bool)
+    PipPkgSpec(name, version = version, binary = binary, extras = extras, editable = editable)
 end
 
 function write_meta(io::IO, meta::Meta)
@@ -102,6 +106,9 @@ end
 function write_meta(io::IO, x::Symbol)
     write_meta(io, String(x))
 end
+function write_meta(io::IO, x::Bool)
+    write(io, x)
+end
 function write_meta(io::IO, x::Vector)
     write(io, convert(Int, length(x)))
     for item in x
@@ -125,4 +132,5 @@ function write_meta(io::IO, x::PipPkgSpec)
     write_meta(io, x.version)
     write_meta(io, x.binary)
     write_meta(io, x.extras)
+    write_meta(io, x.editable)
 end

src/resolve.jl

@@ -437,6 +437,7 @@ function _resolve_merge_pip_packages(packages)
         urls = String[]
         binary = ""
         extras = String[]
+        editables = Bool[]
         for (fn, pkg) in pkgs
             @assert pkg.name == name
             if startswith(pkg.version, "@")
@@ -458,6 +459,7 @@ function _resolve_merge_pip_packages(packages)
                 end
             end
             append!(extras, pkg.extras)
+            push!(editables, pkg.editable)
         end
         sort!(unique!(urls))
         sort!(unique!(versions))
@@ -473,7 +475,12 @@ function _resolve_merge_pip_packages(packages)
                 "direct references ('@ ...') and version specifiers both given for pip package '$name'",
             )
         end
-        push!(specs, PipPkgSpec(name, version = version, binary = binary, extras = extras))
+        unique!(editables)
+        if length(editables) != 1
+            error("both 'editable = true' and 'editable = false' specified for pip package '$name'")
+        end
+        editable = only(editables)
+        push!(specs, PipPkgSpec(name, version = version, binary = binary, extras = extras, editable = editable))
     end
     sort!(specs, by = x -> x.name)
 end
@@ -588,9 +595,16 @@ function _resolve_pip_install(io, pip_specs, load_path, backend)
         elseif spec.binary == "no"
             push!(args, "--no-binary", spec.name)
         end
+        if spec.editable
+            # remove the @ from the beginning of the path.
+            url = replace(spec.version, r"@\s*"=>"")
+            push!(args, "--editable", url)
+        end
     end
     for spec in pip_specs
-        push!(args, specstr(spec))
+        if !spec.editable
+            push!(args, specstr(spec))
+        end
     end
     vrb = _verbosity_flags()
     flags = vrb

src/spec.jl

@@ -133,22 +133,25 @@ struct PipPkgSpec
     version::String
     binary::String
     extras::Vector{String}
-    function PipPkgSpec(name; version = "", binary = "", extras = String[])
+    editable::Bool
+    function PipPkgSpec(name; version = "", binary = "", extras = String[], editable = false)
         name = validate_pip_pkg(name)
         version = validate_pip_version(version)
         binary = validate_pip_binary(binary)
         extras = validate_pip_extras(extras)
-        new(name, version, binary, extras)
+        validate_pip_editable(editable, version)
+        new(name, version, binary, extras, editable)
     end
 end
 
 Base.:(==)(x::PipPkgSpec, y::PipPkgSpec) =
     (x.name == y.name) &&
     (x.version == y.version) &&
     (x.binary == y.binary) &&
-    (x.extras == y.extras)
+    (x.extras == y.extras) &&
+    (x.editable == y.editable)
 Base.hash(x::PipPkgSpec, h::UInt) =
-    hash(x.extras, hash(x.binary, hash(x.version, hash(x.name, h))))
+    hash(x.editable, hash(x.extras, hash(x.binary, hash(x.version, hash(x.name, h)))))
 
 is_valid_pip_pkg(name) = occursin(r"^\s*[-_.A-Za-z0-9]+\s*$", name)
 
@@ -225,8 +228,24 @@ function pixispec(x::PipPkgSpec)
             else
                 spec["git"] = url
             end
+        elseif startswith(url, r"[a-z]+://")
+            # Handle the file:// prefix.
+            m = match(r"^file://(.*)", url)
+            if m !== nothing
+                spec["path"] = m.captures[1]
+                if x.editable
+                    spec["editable"] = true
+                end
+            else
+                spec["url"] = url
+            end
         else
-            spec["url"] = url
+            # https://pixi.sh/latest/reference/pixi_manifest/#path
+            # minimal-project = { path = "./minimal-project", editable = true}
+            spec["path"] = url
+            if x.editable
+                spec["editable"] = true
+            end
         end
     else
         spec["version"] = x.version == "" ? "*" : x.version
@@ -240,3 +259,7 @@ function pixispec(x::PipPkgSpec)
         spec
     end
 end
+validate_pip_editable(editable, version) =
+    if editable && !startswith(version, "@")
+        error("invalid pip version for editable install: must start with `@` but version is $(version)")
+    end

test/Foo/.gitignore

@@ -0,0 +1,3 @@
+*.egg-info
+__pycache__
+build

test/Foo/foo/__init__.py

@@ -0,0 +1 @@
+x = 42

test/Foo/foo/test/added.py

@@ -0,0 +1 @@
+y = 3

test/Foo/setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup(name="foo")

test/FooNonEditable/.gitignore

@@ -0,0 +1,3 @@
+*.egg-info
+__pycache__
+build

test/FooNonEditable/foononeditable/__init__.py

@@ -0,0 +1 @@
+x = -42

test/FooNonEditable/setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup(name="foononeditable")

test/internals.jl

@@ -25,6 +25,8 @@ end
     @test_throws Exception CondaPkg.PipPkgSpec("")
     @test_throws Exception CondaPkg.PipPkgSpec("foo!")
     @test_throws Exception CondaPkg.PipPkgSpec("foo", version = "1.2")
+    @test_throws Exception CondaPkg.PipPkgSpec("foo", editable = true)
+    @test_throws Exception CondaPkg.PipPkgSpec("foo", version = "1.2", editable = true)
     spec = CondaPkg.PipPkgSpec("  F...OO_-0  ", version = "  @./SOME/Path  ")
     @test spec.name == "f-oo-0"
     @test spec.version == "@./SOME/Path"
@@ -36,6 +38,8 @@ end
         CondaPkg.PkgSpec("foo", version = "=1.2.3", channel = "bar"),
         CondaPkg.ChannelSpec("fooo"),
         CondaPkg.PipPkgSpec("foooo", version = "==2.3.4"),
+        CondaPkg.PipPkgSpec("bar", version = "@ /abs/path/somewhere", editable = true),
+        CondaPkg.PipPkgSpec("baz", version = "@ ./rel/path/somewhere", editable = true)
     ]
     for spec in specs
         io = IOBuffer()