Pixi backend

.github/workflows/tests.yml

@@ -18,19 +18,19 @@ jobs:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.backend }} - ${{ github.event_name }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         version:
           - '1'
-          - '1.6'
+          - '1.9'
         os:
           - ubuntu-latest
           - macOS-latest
           - windows-latest
         arch:
           - x64
         backend:
-          - MicroMamba
+          - Pixi
         include:
           - version: '1'
             os: ubuntu-latest
@@ -40,22 +40,34 @@ jobs:
             os: ubuntu-latest
             arch: x64
             backend: 'Null'
+          - version: '1'
+            os: ubuntu-latest
+            arch: x64
+            backend: MicroMamba
+          - version: '1'
+            os: ubuntu-latest
+            arch: x64
+            backend: SystemPixi
     steps:
       - uses: actions/checkout@v3
+      - uses: prefix-dev/setup-pixi@v0
+        if: ${{ matrix.backend == 'SystemPixi' }}
+        with:
+          run-install: false
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
       - uses: julia-actions/cache@v1
       - uses: julia-actions/julia-downgrade-compat@v1
-        if: ${{ matrix.version == '1.6' }}
+        if: ${{ matrix.version == '1.9' }}
         with:
           skip: Markdown,Pkg,TOML,Aqua,Test,TestItemRunner,OpenSSL_jll
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
         env:
           JULIA_CONDAPKG_BACKEND: ${{ matrix.backend }}
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v5
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Unreleased
+* Add `Pixi` and `SystemPixi` backends to allow using [Pixi](https://pixi.sh/latest/) to install packages.
+* The `Pixi` backend is now the default on systems which have it available.
+
 ## 0.2.24 (2024-11-08)
 * Add `pip_backend` preference to choose between `pip` and `uv`.
 * Add `libstdcxx_ng_version` preference to override automatic version bounds.

Project.toml

@@ -10,21 +10,25 @@ MicroMamba = "0b3b1443-0f03-428d-bdfb-f27f9c1191ea"
 Pidfile = "fa939f87-e72e-5be4-a000-7fc836dbe307"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Preferences = "21216c6a-2e73-6563-6e65-726566657250"
+Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+pixi_jll = "4d7b5844-a134-5dcd-ac86-c8f19cd51bed"
 
 [compat]
 Aqua = "0 - 999"
 JSON3 = "1.9"
-Markdown = "1.6"
+Markdown = "1"
 MicroMamba = "0.1.4"
 OpenSSL_jll = "0 - 999"
 Pidfile = "1.3"
-Pkg = "1.6"
+Pkg = "1.9"
 Preferences = "1.3"
+Scratch = "1.2"
+TOML = "1"
 Test = "1"
 TestItemRunner = "0 - 999"
-TOML = "1"
-julia = "1.6"
+julia = "1.9"
+pixi_jll = "0.41.3, 0.42 - 2"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"

README.md

@@ -157,8 +157,8 @@ in more detail.
 
 | Preference | Environment variable | Description |
 | ---------- | -------------------- | ----------- |
-| `backend` | `JULIA_CONDAPKG_BACKEND` | One of `MicroMamba`, `System`, `Current` or `Null` |
-| `exe` | `JULIA_CONDAPKG_EXE` | Path to the Conda executable. |
+| `backend` | `JULIA_CONDAPKG_BACKEND` | One of `MicroMamba`, `Pixi`, `System`, `Current`, `SystemPixi` or `Null` |
+| `exe` | `JULIA_CONDAPKG_EXE` | Path to the Conda/Mamba/MicroMamba/Pixi executable. |
 | `offline` | `JULIA_CONDAPKG_OFFLINE` | When `true`, work in offline mode. |
 | `env` | `JULIA_CONDAPKG_ENV` | Path to the Conda environment to use. |
 | `verbosity` | `JULIA_CONDAPKG_VERBOSITY` | One of `-1`, `0`, `1` or `2`. |
@@ -182,8 +182,13 @@ of Conda is used to manage the Conda environments. You can explicitly select a b
 by setting the `backend` preference to one of the following values:
 - `MicroMamba`: Uses MicroMamba from the package
   [MicroMamba.jl](https://github.com/JuliaPy/MicroMamba.jl).
+- `Pixi`: Uses [Pixi](https://pixi.sh) from
+  [pixi_jll](https://github.com/JuliaBinaryWrappers/pixi_jll.jl) to manage the Conda
+  environments.
 - `System`: Use a pre-installed Conda. If the `exe` preference is set, that is used.
   Otherwise we look for `conda`, `mamba` or `micromamba` in your `PATH`.
+- `SystemPixi`: Use a pre-installed [Pixi](https://pixi.sh). If the `exe` preference
+  is set, that is used. Otherwise we look for `pixi` in your `PATH`.
 - `Current`: Use the currently activated Conda environment instead of creating a new one.
   This backend will only ever install packages, never uninstall. The Conda executable used
   is the same as for the System backend. Similar to the default behaviour of
@@ -192,10 +197,11 @@ by setting the `backend` preference to one of the following values:
   Conda environment that already satisfies the dependencies of your project. It is up to you
   to ensure any required packages are installed.
 
-The default backend is an implementation detail, but is currently `MicroMamba`.
+The default backend is an implementation detail, but is currently `Pixi` or `MicroMamba`,
+depending on your system.
 
-If you set the `exe` preference but not the `backend` preference then the `System` backend
-is used.
+If you set the `exe` preference but not the `backend` preference then the `System` or
+`SystemPixi` backend is used.
 
 ### Offline mode
 

src/CondaPkg.jl

@@ -8,15 +8,30 @@ if isdefined(Base, :Experimental) &&
     @eval Base.Experimental.@compiler_options optimize = 0 infer = false #compile=min
 end
 
-import Base: @kwdef
-import JSON3
-import Pidfile
-import Preferences: @load_preference
-import Pkg
-import TOML
+using Base: @kwdef
+using JSON3: JSON3
+using Pidfile: Pidfile
+using Preferences: @load_preference
+using Pkg: Pkg
+using Scratch: @get_scratch!
+using TOML: TOML
 
-if @load_preference("backend", "MicroMamba") == "MicroMamba"
-    import MicroMamba
+# these are loaded lazily to avoid downloading the JLLs unless they are needed
+MICROMAMBA_MODULE = Ref{Module}()
+PIXI_JLL_MODULE = Ref{Module}()
+
+function micromamba_module()
+    if !isassigned(MICROMAMBA_MODULE)
+        MICROMAMBA_MODULE[] = Base.require(CondaPkg, :MicroMamba)
+    end
+    MICROMAMBA_MODULE[]
+end
+
+function pixi_jll_module()
+    if !isassigned(PIXI_JLL_MODULE)
+        PIXI_JLL_MODULE[] = Base.require(CondaPkg, :pixi_jll)
+    end
+    PIXI_JLL_MODULE[]
 end
 
 let toml = TOML.parsefile(joinpath(@__DIR__, "..", "Project.toml"))
@@ -29,6 +44,7 @@ end
     # backend
     backend::Symbol = :NotSet
     condaexe::String = ""
+    pixiexe::String = ""
     # resolve
     resolved::Bool = false
     load_path::Vector{String} = String[]

src/PkgREPL.jl

@@ -338,8 +338,11 @@ const gc_spec = Pkg.REPLMode.CommandSpec(
 function run(args)
     try
         CondaPkg.withenv() do
-            if args[1] == "conda"
+            b = CondaPkg.backend()
+            if b in CondaPkg.CONDA_BACKENDS && args[1] == "conda"
                 Base.run(CondaPkg.conda_cmd(args[2:end]))
+            elseif b in CondaPkg.PIXI_BACKENDS && args[1] == "pixi"
+                Base.run(CondaPkg.pixi_cmd(args[2:end]))
             else
                 Base.run(Cmd(args))
             end
@@ -357,7 +360,7 @@ conda run cmd ...
 Run the given command in the Conda environment.
 
 You can do `conda run conda ...` to run whichever conda (or mamba or micromamba) executable
-that CondaPkg uses.
+that CondaPkg uses. Or in a pixi backend, do `conda run pixi ...` to run the pixi executable.
 """)
 
 const run_spec = Pkg.REPLMode.CommandSpec(

src/backend.jl

@@ -1,14 +1,44 @@
+"""All valid backends."""
+const ALL_BACKENDS = (:MicroMamba, :Null, :System, :Current, :Pixi, :SystemPixi)
+
+"""All backends that use a Conda/Mamba installer."""
+const CONDA_BACKENDS = (:MicroMamba, :System, :Current)
+
+"""All backends that use a Pixi installer."""
+const PIXI_BACKENDS = (:Pixi, :SystemPixi)
+
 function backend()
     if STATE.backend == :NotSet
         backend = getpref(String, "backend", "JULIA_CONDAPKG_BACKEND", "")
         exe = getpref(String, "exe", "JULIA_CONDAPKG_EXE", "")
+        env = getpref(String, "env", "JULIA_CONDAPKG_ENV", "")
         if backend == ""
-            backend = exe == "" ? "MicroMamba" : "System"
+            if exe == ""
+                if env == "" && invokelatest(pixi_jll_module().is_available)::Bool
+                    # cannot currently use pixi backend if env preference is set
+                    # (see resolve())
+                    backend = "Pixi"
+                elseif invokelatest(micromamba_module().is_available)::Bool
+                    backend = "MicroMamba"
+                else
+                    error(
+                        "neither pixi nor micromamba is automatically available on your system",
+                    )
+                end
+            else
+                if occursin("pixi", lowercase(basename(exe)))
+                    backend = "SystemPixi"
+                else
+                    backend = "System"
+                end
+            end
         end
         if backend == "MicroMamba"
             STATE.backend = :MicroMamba
         elseif backend == "Null"
             STATE.backend = :Null
+        elseif backend == "Pixi"
+            STATE.backend = :Pixi
         elseif backend == "System" || backend == "Current"
             ok = false
             for exe in (exe == "" ? ["micromamba", "mamba", "conda"] : [exe])
@@ -27,24 +57,57 @@ function backend()
                     error("not an executable: $exe")
                 end
             end
+        elseif backend == "SystemPixi"
+            exe2 = Sys.which(exe == "" ? "pixi" : exe)
+            if exe2 === nothing
+                if exe == ""
+                    error("could not find a pixi executable")
+                else
+                    error("not an executable: $exe")
+                end
+            end
+            STATE.backend = :SystemPixi
+            STATE.pixiexe = exe2
         else
             error("invalid backend: $backend")
         end
     end
+    @assert STATE.backend in ALL_BACKENDS
     STATE.backend
 end
 
 function conda_cmd(args = ``; io::IO = stderr)
     b = backend()
     if b == :MicroMamba
-        MicroMamba.cmd(args, io = io)
-    elseif b in (:System, :Current)
+        invokelatest(micromamba_module().cmd, args, io = io)::Cmd
+    elseif b in CONDA_BACKENDS
+        STATE.condaexe == "" && error("this is a bug")
         `$(STATE.condaexe) $args`
-    elseif b == :Null
-        error(
-            "Can not run conda command when backend is Null. Manage conda actions outside of julia.",
-        )
     else
-        @assert false
+        error("Cannot run conda when backend is $b.")
+    end
+end
+
+default_pixi_cache_dir() = @get_scratch!("pixi_cache")
+
+function pixi_cmd(args = ``; io::IO = stderr)
+    b = backend()
+    if b == :Pixi
+        pixiexe = invokelatest(pixi_jll_module().pixi)::Cmd
+        if !haskey(ENV, "PIXI_CACHE_DIR") && !haskey(ENV, "RATTLER_CACHE_DIR")
+            # if the cache dirs are not set, use a scratch dir
+            pixi_cache_dir = default_pixi_cache_dir()
+            pixiexe = addenv(
+                pixiexe,
+                "PIXI_CACHE_DIR" => pixi_cache_dir,
+                "RATTLER_CACHE_DIR" => pixi_cache_dir,
+            )
+        end
+        `$pixiexe $args`
+    elseif b in PIXI_BACKENDS
+        STATE.pixiexe == "" && error("this is a bug")
+        `$(STATE.pixiexe) $args`
+    else
+        error("Cannot run pixi when backend is $b.")
     end
 end

src/deps.jl

@@ -114,15 +114,31 @@ function read_parsed_deps(file)
 end
 
 function current_packages()
-    cmd = conda_cmd(`list -p $(envdir()) --json`)
-    pkglist = JSON3.read(cmd)
+    b = backend()
+    if b in CONDA_BACKENDS
+        cmd = conda_cmd(`list -p $(envdir()) --json`)
+        pkglist = JSON3.read(cmd)
+    elseif b in PIXI_BACKENDS
+        cmd =
+            pixi_cmd(`list --manifest-path $(joinpath(STATE.meta_dir, "pixi.toml")) --json`)
+        pkglist = JSON3.read(cmd)
+        pkglist = [pkg for pkg in pkglist if pkg.kind == "conda"]
+    end
     Dict(normalise_pkg(pkg.name) => pkg for pkg in pkglist)
 end
 
 function current_pip_packages()
-    pkglist = withenv() do
-        cmd = `$(which("pip")) list --format=json`
-        JSON3.read(cmd)
+    b = backend()
+    if b in CONDA_BACKENDS
+        pkglist = withenv() do
+            cmd = `$(which("pip")) list --format=json`
+            JSON3.read(cmd)
+        end
+    elseif b in PIXI_BACKENDS
+        cmd =
+            pixi_cmd(`list --manifest-path $(joinpath(STATE.meta_dir, "pixi.toml")) --json`)
+        pkglist = JSON3.read(cmd)
+        pkglist = [pkg for pkg in pkglist if pkg.kind == "pypi"]
     end
     Dict(normalise_pip_pkg(pkg.name) => pkg for pkg in pkglist)
 end

src/env.jl

@@ -15,8 +15,8 @@ function activate!(e)
     path_sep = Sys.iswindows() ? ';' : ':'
     new_path = join(bindirs(), path_sep)
     if backend() == :MicroMamba
-        e["MAMBA_ROOT_PREFIX"] = MicroMamba.root_dir()
-        new_path = "$(new_path)$(path_sep)$(dirname(MicroMamba.executable()))"
+        e["MAMBA_ROOT_PREFIX"] = invokelatest(micromamba_module().root_dir)::String
+        new_path = "$(new_path)$(path_sep)$(dirname(invokelatest(micromamba_module().executable)::String))"
     end
     if old_path != ""
         new_path = "$(new_path)$(path_sep)$(old_path)"
@@ -119,9 +119,13 @@ end
 Remove unused packages and caches.
 """
 function gc(; io::IO = stderr)
-    backend() == :Null && return
-    resolve()
-    cmd = conda_cmd(`clean -y --all`, io = io)
-    _run(io, cmd, "Removing unused caches")
+    b = backend()
+    if b in CONDA_BACKENDS
+        resolve()
+        cmd = conda_cmd(`clean -y --all`, io = io)
+        _run(io, cmd, "Removing unused caches")
+    else
+        _log(io, "GC does nothing with the $b backend.")
+    end
     return
 end