diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f8dc601..c587d44 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,11 +18,11 @@ jobs: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.backend }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} strategy: - fail-fast: false + fail-fast: true matrix: version: - '1' - - '1.6' + - '1.9' os: - ubuntu-latest - macOS-latest @@ -30,7 +30,7 @@ jobs: arch: - x64 backend: - - MicroMamba + - Pixi include: - version: '1' os: ubuntu-latest @@ -40,15 +40,27 @@ jobs: os: ubuntu-latest arch: x64 backend: 'Null' + - version: '1' + os: ubuntu-latest + arch: x64 + backend: MicroMamba + - version: '1' + os: ubuntu-latest + arch: x64 + backend: SystemPixi steps: - uses: actions/checkout@v3 + - uses: prefix-dev/setup-pixi@v0 + if: ${{ matrix.backend == 'SystemPixi' }} + with: + run-install: false - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - uses: julia-actions/cache@v1 - uses: julia-actions/julia-downgrade-compat@v1 - if: ${{ matrix.version == '1.6' }} + if: ${{ matrix.version == '1.9' }} with: skip: Markdown,Pkg,TOML,Aqua,Test,TestItemRunner,OpenSSL_jll - uses: julia-actions/julia-buildpkg@v1 @@ -56,6 +68,6 @@ jobs: env: JULIA_CONDAPKG_BACKEND: ${{ matrix.backend }} - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v5 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index bf90e6a..d824a38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased +* Add `Pixi` and `SystemPixi` backends to allow using [Pixi](https://pixi.sh/latest/) to install packages. +* The `Pixi` backend is now the default on systems which have it available. + ## 0.2.24 (2024-11-08) * Add `pip_backend` preference to choose between `pip` and `uv`. * Add `libstdcxx_ng_version` preference to override automatic version bounds. diff --git a/Project.toml b/Project.toml index 5dcf576..bade949 100644 --- a/Project.toml +++ b/Project.toml @@ -10,21 +10,25 @@ MicroMamba = "0b3b1443-0f03-428d-bdfb-f27f9c1191ea" Pidfile = "fa939f87-e72e-5be4-a000-7fc836dbe307" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Preferences = "21216c6a-2e73-6563-6e65-726566657250" +Scratch = "6c6a2e73-6563-6170-7368-637461726353" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +pixi_jll = "4d7b5844-a134-5dcd-ac86-c8f19cd51bed" [compat] Aqua = "0 - 999" JSON3 = "1.9" -Markdown = "1.6" +Markdown = "1" MicroMamba = "0.1.4" OpenSSL_jll = "0 - 999" Pidfile = "1.3" -Pkg = "1.6" +Pkg = "1.9" Preferences = "1.3" +Scratch = "1.2" +TOML = "1" Test = "1" TestItemRunner = "0 - 999" -TOML = "1" -julia = "1.6" +julia = "1.9" +pixi_jll = "0.41.3, 0.42 - 2" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" diff --git a/README.md b/README.md index cb34443..bb69932 100644 --- a/README.md +++ b/README.md @@ -157,8 +157,8 @@ in more detail. | Preference | Environment variable | Description | | ---------- | -------------------- | ----------- | -| `backend` | `JULIA_CONDAPKG_BACKEND` | One of `MicroMamba`, `System`, `Current` or `Null` | -| `exe` | `JULIA_CONDAPKG_EXE` | Path to the Conda executable. | +| `backend` | `JULIA_CONDAPKG_BACKEND` | One of `MicroMamba`, `Pixi`, `System`, `Current`, `SystemPixi` or `Null` | +| `exe` | `JULIA_CONDAPKG_EXE` | Path to the Conda/Mamba/MicroMamba/Pixi executable. | | `offline` | `JULIA_CONDAPKG_OFFLINE` | When `true`, work in offline mode. | | `env` | `JULIA_CONDAPKG_ENV` | Path to the Conda environment to use. | | `verbosity` | `JULIA_CONDAPKG_VERBOSITY` | One of `-1`, `0`, `1` or `2`. | @@ -182,8 +182,13 @@ of Conda is used to manage the Conda environments. You can explicitly select a b by setting the `backend` preference to one of the following values: - `MicroMamba`: Uses MicroMamba from the package [MicroMamba.jl](https://github.com/JuliaPy/MicroMamba.jl). +- `Pixi`: Uses [Pixi](https://pixi.sh) from + [pixi_jll](https://github.com/JuliaBinaryWrappers/pixi_jll.jl) to manage the Conda + environments. - `System`: Use a pre-installed Conda. If the `exe` preference is set, that is used. Otherwise we look for `conda`, `mamba` or `micromamba` in your `PATH`. +- `SystemPixi`: Use a pre-installed [Pixi](https://pixi.sh). If the `exe` preference + is set, that is used. Otherwise we look for `pixi` in your `PATH`. - `Current`: Use the currently activated Conda environment instead of creating a new one. This backend will only ever install packages, never uninstall. The Conda executable used is the same as for the System backend. Similar to the default behaviour of @@ -192,10 +197,11 @@ by setting the `backend` preference to one of the following values: Conda environment that already satisfies the dependencies of your project. It is up to you to ensure any required packages are installed. -The default backend is an implementation detail, but is currently `MicroMamba`. +The default backend is an implementation detail, but is currently `Pixi` or `MicroMamba`, +depending on your system. -If you set the `exe` preference but not the `backend` preference then the `System` backend -is used. +If you set the `exe` preference but not the `backend` preference then the `System` or +`SystemPixi` backend is used. ### Offline mode diff --git a/src/CondaPkg.jl b/src/CondaPkg.jl index dc5f60c..ce9f40c 100644 --- a/src/CondaPkg.jl +++ b/src/CondaPkg.jl @@ -8,15 +8,30 @@ if isdefined(Base, :Experimental) && @eval Base.Experimental.@compiler_options optimize = 0 infer = false #compile=min end -import Base: @kwdef -import JSON3 -import Pidfile -import Preferences: @load_preference -import Pkg -import TOML +using Base: @kwdef +using JSON3: JSON3 +using Pidfile: Pidfile +using Preferences: @load_preference +using Pkg: Pkg +using Scratch: @get_scratch! +using TOML: TOML -if @load_preference("backend", "MicroMamba") == "MicroMamba" - import MicroMamba +# these are loaded lazily to avoid downloading the JLLs unless they are needed +MICROMAMBA_MODULE = Ref{Module}() +PIXI_JLL_MODULE = Ref{Module}() + +function micromamba_module() + if !isassigned(MICROMAMBA_MODULE) + MICROMAMBA_MODULE[] = Base.require(CondaPkg, :MicroMamba) + end + MICROMAMBA_MODULE[] +end + +function pixi_jll_module() + if !isassigned(PIXI_JLL_MODULE) + PIXI_JLL_MODULE[] = Base.require(CondaPkg, :pixi_jll) + end + PIXI_JLL_MODULE[] end let toml = TOML.parsefile(joinpath(@__DIR__, "..", "Project.toml")) @@ -29,6 +44,7 @@ end # backend backend::Symbol = :NotSet condaexe::String = "" + pixiexe::String = "" # resolve resolved::Bool = false load_path::Vector{String} = String[] diff --git a/src/PkgREPL.jl b/src/PkgREPL.jl index 2c4e621..1d2099a 100644 --- a/src/PkgREPL.jl +++ b/src/PkgREPL.jl @@ -338,8 +338,11 @@ const gc_spec = Pkg.REPLMode.CommandSpec( function run(args) try CondaPkg.withenv() do - if args[1] == "conda" + b = CondaPkg.backend() + if b in CondaPkg.CONDA_BACKENDS && args[1] == "conda" Base.run(CondaPkg.conda_cmd(args[2:end])) + elseif b in CondaPkg.PIXI_BACKENDS && args[1] == "pixi" + Base.run(CondaPkg.pixi_cmd(args[2:end])) else Base.run(Cmd(args)) end @@ -357,7 +360,7 @@ conda run cmd ... Run the given command in the Conda environment. You can do `conda run conda ...` to run whichever conda (or mamba or micromamba) executable -that CondaPkg uses. +that CondaPkg uses. Or in a pixi backend, do `conda run pixi ...` to run the pixi executable. """) const run_spec = Pkg.REPLMode.CommandSpec( diff --git a/src/backend.jl b/src/backend.jl index 1723aff..4a8579b 100644 --- a/src/backend.jl +++ b/src/backend.jl @@ -1,14 +1,44 @@ +"""All valid backends.""" +const ALL_BACKENDS = (:MicroMamba, :Null, :System, :Current, :Pixi, :SystemPixi) + +"""All backends that use a Conda/Mamba installer.""" +const CONDA_BACKENDS = (:MicroMamba, :System, :Current) + +"""All backends that use a Pixi installer.""" +const PIXI_BACKENDS = (:Pixi, :SystemPixi) + function backend() if STATE.backend == :NotSet backend = getpref(String, "backend", "JULIA_CONDAPKG_BACKEND", "") exe = getpref(String, "exe", "JULIA_CONDAPKG_EXE", "") + env = getpref(String, "env", "JULIA_CONDAPKG_ENV", "") if backend == "" - backend = exe == "" ? "MicroMamba" : "System" + if exe == "" + if env == "" && invokelatest(pixi_jll_module().is_available)::Bool + # cannot currently use pixi backend if env preference is set + # (see resolve()) + backend = "Pixi" + elseif invokelatest(micromamba_module().is_available)::Bool + backend = "MicroMamba" + else + error( + "neither pixi nor micromamba is automatically available on your system", + ) + end + else + if occursin("pixi", lowercase(basename(exe))) + backend = "SystemPixi" + else + backend = "System" + end + end end if backend == "MicroMamba" STATE.backend = :MicroMamba elseif backend == "Null" STATE.backend = :Null + elseif backend == "Pixi" + STATE.backend = :Pixi elseif backend == "System" || backend == "Current" ok = false for exe in (exe == "" ? ["micromamba", "mamba", "conda"] : [exe]) @@ -27,24 +57,57 @@ function backend() error("not an executable: $exe") end end + elseif backend == "SystemPixi" + exe2 = Sys.which(exe == "" ? "pixi" : exe) + if exe2 === nothing + if exe == "" + error("could not find a pixi executable") + else + error("not an executable: $exe") + end + end + STATE.backend = :SystemPixi + STATE.pixiexe = exe2 else error("invalid backend: $backend") end end + @assert STATE.backend in ALL_BACKENDS STATE.backend end function conda_cmd(args = ``; io::IO = stderr) b = backend() if b == :MicroMamba - MicroMamba.cmd(args, io = io) - elseif b in (:System, :Current) + invokelatest(micromamba_module().cmd, args, io = io)::Cmd + elseif b in CONDA_BACKENDS + STATE.condaexe == "" && error("this is a bug") `$(STATE.condaexe) $args` - elseif b == :Null - error( - "Can not run conda command when backend is Null. Manage conda actions outside of julia.", - ) else - @assert false + error("Cannot run conda when backend is $b.") + end +end + +default_pixi_cache_dir() = @get_scratch!("pixi_cache") + +function pixi_cmd(args = ``; io::IO = stderr) + b = backend() + if b == :Pixi + pixiexe = invokelatest(pixi_jll_module().pixi)::Cmd + if !haskey(ENV, "PIXI_CACHE_DIR") && !haskey(ENV, "RATTLER_CACHE_DIR") + # if the cache dirs are not set, use a scratch dir + pixi_cache_dir = default_pixi_cache_dir() + pixiexe = addenv( + pixiexe, + "PIXI_CACHE_DIR" => pixi_cache_dir, + "RATTLER_CACHE_DIR" => pixi_cache_dir, + ) + end + `$pixiexe $args` + elseif b in PIXI_BACKENDS + STATE.pixiexe == "" && error("this is a bug") + `$(STATE.pixiexe) $args` + else + error("Cannot run pixi when backend is $b.") end end diff --git a/src/deps.jl b/src/deps.jl index 9ad4d38..8e84e4a 100644 --- a/src/deps.jl +++ b/src/deps.jl @@ -114,15 +114,31 @@ function read_parsed_deps(file) end function current_packages() - cmd = conda_cmd(`list -p $(envdir()) --json`) - pkglist = JSON3.read(cmd) + b = backend() + if b in CONDA_BACKENDS + cmd = conda_cmd(`list -p $(envdir()) --json`) + pkglist = JSON3.read(cmd) + elseif b in PIXI_BACKENDS + cmd = + pixi_cmd(`list --manifest-path $(joinpath(STATE.meta_dir, "pixi.toml")) --json`) + pkglist = JSON3.read(cmd) + pkglist = [pkg for pkg in pkglist if pkg.kind == "conda"] + end Dict(normalise_pkg(pkg.name) => pkg for pkg in pkglist) end function current_pip_packages() - pkglist = withenv() do - cmd = `$(which("pip")) list --format=json` - JSON3.read(cmd) + b = backend() + if b in CONDA_BACKENDS + pkglist = withenv() do + cmd = `$(which("pip")) list --format=json` + JSON3.read(cmd) + end + elseif b in PIXI_BACKENDS + cmd = + pixi_cmd(`list --manifest-path $(joinpath(STATE.meta_dir, "pixi.toml")) --json`) + pkglist = JSON3.read(cmd) + pkglist = [pkg for pkg in pkglist if pkg.kind == "pypi"] end Dict(normalise_pip_pkg(pkg.name) => pkg for pkg in pkglist) end diff --git a/src/env.jl b/src/env.jl index 19dd705..06795ff 100644 --- a/src/env.jl +++ b/src/env.jl @@ -15,8 +15,8 @@ function activate!(e) path_sep = Sys.iswindows() ? ';' : ':' new_path = join(bindirs(), path_sep) if backend() == :MicroMamba - e["MAMBA_ROOT_PREFIX"] = MicroMamba.root_dir() - new_path = "$(new_path)$(path_sep)$(dirname(MicroMamba.executable()))" + e["MAMBA_ROOT_PREFIX"] = invokelatest(micromamba_module().root_dir)::String + new_path = "$(new_path)$(path_sep)$(dirname(invokelatest(micromamba_module().executable)::String))" end if old_path != "" new_path = "$(new_path)$(path_sep)$(old_path)" @@ -119,9 +119,13 @@ end Remove unused packages and caches. """ function gc(; io::IO = stderr) - backend() == :Null && return - resolve() - cmd = conda_cmd(`clean -y --all`, io = io) - _run(io, cmd, "Removing unused caches") + b = backend() + if b in CONDA_BACKENDS + resolve() + cmd = conda_cmd(`clean -y --all`, io = io) + _run(io, cmd, "Removing unused caches") + else + _log(io, "GC does nothing with the $b backend.") + end return end diff --git a/src/meta.jl b/src/meta.jl index becd895..5902169 100644 --- a/src/meta.jl +++ b/src/meta.jl @@ -4,7 +4,7 @@ information about the most recent resolve. """ # increment whenever the metadata format changes -const META_VERSION = 13 +const META_VERSION = 14 @kwdef mutable struct Meta timestamp::Float64 @@ -12,6 +12,7 @@ const META_VERSION = 13 load_path::Vector{String} extra_path::Vector{String} version::VersionNumber + backend::Symbol packages::Vector{PkgSpec} channels::Vector{ChannelSpec} pip_packages::Vector{PipPkgSpec} @@ -26,6 +27,7 @@ function read_meta(io::IO) load_path = read_meta(io, Vector{String}), extra_path = read_meta(io, Vector{String}), version = read_meta(io, VersionNumber), + backend = read_meta(io, Symbol), packages = read_meta(io, Vector{PkgSpec}), channels = read_meta(io, Vector{ChannelSpec}), pip_packages = read_meta(io, Vector{PipPkgSpec}), @@ -43,6 +45,9 @@ function read_meta(io::IO, ::Type{String}) end String(bytes) end +function read_meta(io::IO, ::Type{Symbol}) + Symbol(read_meta(io, String)) +end function read_meta(io::IO, ::Type{Vector{T}}) where {T} len = read(io, Int) ans = Vector{T}() @@ -81,6 +86,7 @@ function write_meta(io::IO, meta::Meta) write_meta(io, meta.load_path) write_meta(io, meta.extra_path) write_meta(io, meta.version) + write_meta(io, meta.backend) write_meta(io, meta.packages) write_meta(io, meta.channels) write_meta(io, meta.pip_packages) @@ -93,6 +99,9 @@ function write_meta(io::IO, x::String) write(io, convert(Int, sizeof(x))) write(io, x) end +function write_meta(io::IO, x::Symbol) + write_meta(io, String(x)) +end function write_meta(io::IO, x::Vector) write(io, convert(Int, length(x))) for item in x diff --git a/src/resolve.jl b/src/resolve.jl index 2edc6d5..5ac6e63 100644 --- a/src/resolve.jl +++ b/src/resolve.jl @@ -52,6 +52,10 @@ function _resolve_can_skip_1(conda_env, load_path, meta_file) @debug "conda env has changed" meta.conda_env conda_env return false end + if meta.backend != backend() + @debug "backend has changed" meta.backend backend() + return false + end timestamp = max(meta.timestamp, stat(meta_file).mtime) for env in [meta.load_path; meta.extra_path] dir = isfile(env) ? dirname(env) : isdir(env) ? env : continue @@ -256,12 +260,44 @@ function _resolve_merge_packages(packages, channels) end end end - for pkg in values(pkgs) + # now merge packages with the same name + version = "" + channel = "" + build = "" + for (fn, pkg) in pkgs @assert pkg.name == name - push!(specs, pkg) + if pkg.version != "" + if version == "" + version = pkg.version + else + version = _resolve_merge_versions(version, pkg.version) + end + end + if pkg.channel != "" + if channel == "" + channel = pkg.channel + elseif channel != pkg.channel + error("multiple channels specified for package '$name'") + end + end + if pkg.build != "" + if build == "" + build = pkg.build + elseif build != pkg.build + error("multiple builds specified for package '$name'") + end + end end + push!(specs, PkgSpec(name, version = version, channel = channel, build = build)) end - sort!(unique!(specs), by = x -> x.name) + sort!(specs, by = x -> x.name) +end + +function _resolve_merge_versions(v1, v2) + parts1 = split(v1, "|") + parts2 = split(v2, "|") + parts = ["$(strip(part1)), $(strip(part2))" for part1 in parts1 for part2 in parts2] + join(parts, " | ") end function abspathurl(args...) @@ -495,17 +531,22 @@ function _cmdlines(cmd, flags) lines end +function _logblock(io::IO, lines; kw...) + lines = collect(String, lines) + for (i, line) in enumerate(lines) + pre = i == length(lines) ? "└ " : "│ " + _log(io, label = "") do io + print(io, pre) + printstyled(io, line; kw...) + end + end +end + function _run(io::IO, cmd::Cmd, args...; flags = String[]) _log(io, args...) if _verbosity() ≥ 0 lines = _cmdlines(cmd, flags) - for (i, line) in enumerate(lines) - pre = i == length(lines) ? "└ " : "│ " - _log(io, label = "") do io - print(io, pre) - printstyled(io, line, color = :light_black) - end - end + _logblock(io, lines, color = :light_black) end run(cmd) end @@ -562,8 +603,16 @@ function resolve(; ) shared = true elseif conda_env == "" - conda_env = joinpath(meta_dir, "env") + if back in CONDA_BACKENDS + conda_env = joinpath(meta_dir, "env") + elseif back in PIXI_BACKENDS + conda_env = joinpath(meta_dir, ".pixi", "envs", "default") + else + error("this is a bug") + end shared = false + elseif !(back in CONDA_BACKENDS) + error("cannot set env preference with $back backend") elseif startswith(conda_env, "@") conda_env_name = conda_env[2:end] conda_env_name == "" && error("shared env name cannot be empty") @@ -671,55 +720,118 @@ function resolve(; _log(io, char, " ", pkg, label = "", color = color) end end - # install/uninstall packages - if !force && meta !== nothing && _resolve_env_is_clean(conda_env, meta) - # the state is sufficiently clean that we can modify the existing conda environment - changed = false - if !isempty(removed_pip_pkgs) && !shared - dry_run && return - changed = true - _resolve_pip_remove(io, removed_pip_pkgs, load_path, pip_backend) - end - if !isempty(removed_pkgs) && !shared - dry_run && return - changed = true - _resolve_conda_remove(io, conda_env, removed_pkgs) - end - if !isempty(specs) && ( - !isempty(added_pkgs) || - !isempty(changed_pkgs) || - (meta.channels != channels) || - changed - ) - dry_run && return - changed = true - _resolve_conda_install(io, conda_env, specs, channels) - end - if !isempty(pip_specs) && - (!isempty(added_pip_pkgs) || !isempty(changed_pip_pkgs) || changed) + if back in CONDA_BACKENDS + # install/uninstall packages + if !force && meta !== nothing && _resolve_env_is_clean(conda_env, meta) + # the state is sufficiently clean that we can modify the existing conda environment + changed = false + if !isempty(removed_pip_pkgs) && !shared + dry_run && return + changed = true + _resolve_pip_remove(io, removed_pip_pkgs, load_path, pip_backend) + end + if !isempty(removed_pkgs) && !shared + dry_run && return + changed = true + _resolve_conda_remove(io, conda_env, removed_pkgs) + end + if !isempty(specs) && ( + !isempty(added_pkgs) || + !isempty(changed_pkgs) || + (meta.channels != channels) || + changed + ) + dry_run && return + changed = true + _resolve_conda_install(io, conda_env, specs, channels) + end + if !isempty(pip_specs) && + (!isempty(added_pip_pkgs) || !isempty(changed_pip_pkgs) || changed) + dry_run && return + changed = true + _resolve_pip_install(io, pip_specs, load_path, pip_backend) + end + changed || _log(io, "Dependencies already up to date") + else + # the state is too dirty, recreate the conda environment from scratch dry_run && return - changed = true - _resolve_pip_install(io, pip_specs, load_path, pip_backend) + # remove environment + mkpath(meta_dir) + create = true + if isdir(conda_env) + if shared + create = false + else + _resolve_conda_remove_all(io, conda_env) + end + end + # create conda environment + _resolve_conda_install(io, conda_env, specs, channels; create = create) + # install pip packages + isempty(pip_specs) || + _resolve_pip_install(io, pip_specs, load_path, pip_backend) end - changed || _log(io, "Dependencies already up to date") - else - # the state is too dirty, recreate the conda environment from scratch + elseif back in PIXI_BACKENDS dry_run && return - # remove environment - mkpath(meta_dir) - create = true - if isdir(conda_env) - if shared - create = false - else - _resolve_conda_remove_all(io, conda_env) + cd(meta_dir) do + # remove existing files that might confuse pixi + Base.rm(joinpath(meta_dir, "pixi.toml"), force = true) + Base.rm(joinpath(meta_dir, "pyproject.toml"), force = true) + force && Base.rm(joinpath(meta_dir, "pixi.lock"), force = true) + # write .pixi/config.toml + configtomlpath = joinpath(meta_dir, ".pixi", "config.toml") + configtoml = Dict{String,Any}("detached-environments" => false) + configtomlstr = sprint(TOML.print, configtoml) + mkpath(dirname(configtomlpath)) + write(configtomlpath, configtomlstr) + # initialise pixi + _run( + io, + pixi_cmd(`init --format pixi $meta_dir`), + "Initialising pixi", + flags = ["--quiet"], + ) + # load pixi.toml + pixitomlpath = joinpath(meta_dir, "pixi.toml") + pixitoml = open(TOML.parse, pixitomlpath) + # new pixi.toml + pixitoml = Dict{String,Any}( + "project" => Dict{String,Any}( + "name" => ".CondaPkg", + "description" => "automatically generated by CondaPkg.jl", + "platforms" => pixitoml["project"]["platforms"], + "channels" => String[specstr(channel) for channel in channels], + "channel-priority" => "disabled", + ), + "dependencies" => + Dict{String,Any}(spec.name => pixispec(spec) for spec in specs), + ) + if !isempty(pip_specs) + pixitoml["pypi-dependencies"] = + Dict{String,Any}(spec.name => pixispec(spec) for spec in pip_specs) end + for spec in pip_specs + if spec.binary != "" + _log( + io, + "Warning: $b backend ignoring binary=$(spec.binary) for $(spec.name)", + ) + end + end + pixitomlstr = sprint(TOML.print, pixitoml) + write(pixitomlpath, pixitomlstr) + _log(io, "Wrote $pixitomlpath") + _logblock(io, eachline(pixitomlpath), color = :light_black) + _run( + io, + pixi_cmd( + `$(force ? "update" : "install") --manifest-path $pixitomlpath`, + ), + "Installing packages", + ) end - # create conda environment - _resolve_conda_install(io, conda_env, specs, channels; create = create) - # install pip packages - isempty(pip_specs) || - _resolve_pip_install(io, pip_specs, load_path, pip_backend) + else + error("this is a bug") end # save metadata meta = Meta( @@ -728,6 +840,7 @@ function resolve(; load_path = load_path, extra_path = extra_path, version = VERSION, + backend = back, packages = specs, channels = channels, pip_packages = pip_specs, diff --git a/src/spec.jl b/src/spec.jl index da070b0..216793d 100644 --- a/src/spec.jl +++ b/src/spec.jl @@ -87,6 +87,22 @@ function specstr(x::PkgSpec) string(x.name, suffix) end +function pixispec(x::PkgSpec) + spec = Dict{String,Any}() + spec["version"] = x.version == "" ? "*" : x.version + if x.build != "" + spec["build"] = x.build + end + if x.channel != "" + spec["channel"] = x.channel + end + if length(spec) == 1 + spec["version"] + else + spec + end +end + struct ChannelSpec name::String function ChannelSpec(name) @@ -190,3 +206,37 @@ function specstr(x::PipPkgSpec) end return join(parts) end + +function pixispec(x::PipPkgSpec) + spec = Dict{String,Any}() + if startswith(x.version, "@") + url = strip(x.version[2:end]) + if startswith(url, "git+") + url = url[5:end] + if (m = match(r"^(.*?)@([^/@]*)$", url); m !== nothing) + spec["git"] = m.captures[1] + rev = m.captures[2] + if (m = match(r"^([^#]*)#(.*)$", rev); m !== nothing) + # spec["tag"] = m.captures[1] + spec["rev"] = m.captures[2] + else + spec["rev"] = rev + end + else + spec["git"] = url + end + else + spec["url"] = url + end + else + spec["version"] = x.version == "" ? "*" : x.version + end + if !isempty(x.extras) + spec["extras"] = x.extras + end + if length(spec) == 1 && haskey(spec, "version") + spec["version"] + else + spec + end +end diff --git a/test/aqua.jl b/test/aqua.jl index eea73ca..4a89a98 100644 --- a/test/aqua.jl +++ b/test/aqua.jl @@ -1,4 +1,8 @@ @testitem "Aqua" begin import Aqua - Aqua.test_all(CondaPkg) + Aqua.test_all( + CondaPkg; + # these are loaded lazily + stale_deps = (; ignore = [:MicroMamba, :pixi_jll]), + ) end diff --git a/test/internals.jl b/test/internals.jl index 17377c7..5518e14 100644 --- a/test/internals.jl +++ b/test/internals.jl @@ -60,6 +60,19 @@ end end end +@testitem "_resolve_merge_versions" begin + include("setup.jl") + @testset "$(case.v1) $(case.v2)" for case in [ + (v1 = "1.2.3", v2 = "1.2.3", expected = "1.2.3, 1.2.3"), + (v1 = "1.2.3", v2 = "1.2.4", expected = "1.2.3, 1.2.4"), + (v1 = "1.2.3", v2 = ">=1.2,<2", expected = "1.2.3, >=1.2,<2"), + (v1 = ">=1.2,<2", v2 = ">=2.1,<3", expected = ">=1.2,<2, >=2.1,<3"), + (v1 = "1|2", v2 = "2|3", expected = "1, 2 | 1, 3 | 2, 2 | 2, 3"), + ] + @test CondaPkg._resolve_merge_versions(case.v1, case.v2) == case.expected + end +end + @testitem "_compatible_libstdcxx_ng_version" begin include("setup.jl") key = "JULIA_CONDAPKG_LIBSTDCXX_NG_VERSION" diff --git a/test/main.jl b/test/main.jl index 6c7d996..b76c12e 100644 --- a/test/main.jl +++ b/test/main.jl @@ -166,14 +166,17 @@ end @testitem "install/remove executable package" begin include("setup.jl") if !isnull - CondaPkg.add("curl", resolve = false) + CondaPkg.add("uv", resolve = false) CondaPkg.resolve(force = true) - curl_path = CondaPkg.which("curl") - @test curl_path !== nothing - @test isfile(curl_path) - CondaPkg.rm("curl", resolve = false) + exe_path = CondaPkg.which("uv") + @test exe_path !== nothing + @test isfile(exe_path) + CondaPkg.rm("uv", resolve = false) CondaPkg.resolve(force = true) - @test !isfile(curl_path) + if !ispixi + # pixi doesn't seem to remove unused packages?? + @test !isfile(exe_path) + end end end @@ -198,7 +201,7 @@ end @testitem "external conda env" begin include("setup.jl") dn = string(tempname(), backend, Sys.KERNEL, VERSION) - isnull || withenv("JULIA_CONDAPKG_ENV" => dn) do + (isnull || ispixi) || withenv("JULIA_CONDAPKG_ENV" => dn) do # create empty env CondaPkg.resolve() @test !occursin("ca-certificates", status()) @@ -219,13 +222,13 @@ end @testitem "shared env" begin include("setup.jl") - isnull || withenv("JULIA_CONDAPKG_ENV" => "@my_env") do + (isnull || ispixi) || withenv("JULIA_CONDAPKG_ENV" => "@my_env") do CondaPkg.add("python"; force = true) @test CondaPkg.envdir() == joinpath(Base.DEPOT_PATH[1], "conda_environments", "my_env") @test isfile(CondaPkg.envdir(Sys.iswindows() ? "python.exe" : "bin/python")) end - isnull || withenv("JULIA_CONDAPKG_ENV" => "@/some/absolute/path") do + (isnull || ispixi) || withenv("JULIA_CONDAPKG_ENV" => "@/some/absolute/path") do @test_throws ErrorException CondaPkg.add("python"; force = true) end end diff --git a/test/pkgrepl.jl b/test/pkgrepl.jl index cf2efb3..75946f0 100644 --- a/test/pkgrepl.jl +++ b/test/pkgrepl.jl @@ -69,14 +69,12 @@ end @test contains(read(fn, String), "3.10.2") # run conda --help and check the output # tests that conda and mamba both run whatever CondaPkg runs - for cmd in ["conda", "mamba"] - open(fn, "w") do io - redirect_stdout(io) do - CondaPkg.PkgREPL.run(["conda", "--help"]) - end + open(fn, "w") do io + redirect_stdout(io) do + CondaPkg.PkgREPL.run([ispixi ? "pixi" : "conda", "--help"]) end - @test contains(read(fn, String), "--help") - @test contains(read(fn, String), "--version") end + @test contains(read(fn, String), "--help") + @test contains(read(fn, String), "--version") end end diff --git a/test/setup.jl b/test/setup.jl index 31d7094..191fe36 100644 --- a/test/setup.jl +++ b/test/setup.jl @@ -6,14 +6,16 @@ ENV["JULIA_CONDAPKG_VERBOSITY"] = "0" status() = sprint(io -> CondaPkg.status(io = io)) -const backend = get(ENV, "JULIA_CONDAPKG_BACKEND", "MicroMamba") +const backend = get(ENV, "JULIA_CONDAPKG_BACKEND", "Pixi") const isnull = backend == "Null" +const ispixi = backend == "SystemPixi" || backend == "Pixi" # reset the package state (so tests are independent of the order they are run) rm(CondaPkg.cur_deps_file(), force = true) CondaPkg.STATE.backend = :NotSet CondaPkg.STATE.condaexe = "" +CondaPkg.STATE.pixiexe = "" CondaPkg.STATE.resolved = false CondaPkg.STATE.load_path = String[] CondaPkg.STATE.meta_dir = ""