Add uv process directive for Python package management

docs/reference/config.md

@@ -1603,6 +1603,29 @@ The following settings are available:
 `spack.parallelBuilds`
 : The maximum number of parallel package builds (default: the number of available CPUs).
 
+(config-uv)=
+
+## `uv`
+
+The `uv` scope controls the creation of Python virtual environments by the [uv](https://docs.astral.sh/uv/) package manager.
+
+The following settings are available:
+
+`uv.cacheDir`
+: The path where uv virtual environments are stored. It should be accessible from all compute nodes when using a shared file system.
+
+`uv.createTimeout`
+: The amount of time to wait for the uv environment to be created before failing (default: `20 min`).
+
+`uv.enabled`
+: Execute tasks with uv virtual environments (default: `false`).
+
+`uv.installOptions`
+: Extra command line options for the `uv pip install` command. See the [uv documentation](https://docs.astral.sh/uv/) for more information.
+
+`uv.pythonVersion`
+: The Python version to use when creating virtual environments (e.g. `3.12`). If not specified, uv will use its default Python resolution.
+
 (config-timeline)=
 
 ## `timeline`

docs/reference/env-vars.md

@@ -232,6 +232,12 @@ The following environment variables control the configuration of the Nextflow ru
   :::
 : Enable the use of Spack recipes defined by using the {ref}`process-spack` directive. (default: `false`).
 
+`NXF_UV_CACHEDIR`
+: Directory where uv virtual environments are stored. When using a computing cluster it must be a shared folder accessible from all compute nodes.
+
+`NXF_UV_ENABLED`
+: Enable the use of uv environments defined by using the {ref}`process-uv` directive. (default: `false`).
+
 `NXF_SYNTAX_PARSER`
 : :::{versionadded} 25.02.0-edge
   :::

docs/reference/process.md

@@ -1574,6 +1574,35 @@ Multiple packages can be specified separating them with a blank space, e.g. `bwa
 
 The `spack` directive also accepts a Spack environment file path or the path of an existing Spack environment. See {ref}`spack-page` for more information.
 
+(process-uv)=
+
+### uv
+
+The `uv` directive defines the set of Python packages to be installed using the [uv](https://docs.astral.sh/uv/) package manager for each task. For example:
+
+```nextflow
+process hello {
+    uv 'numpy pandas matplotlib'
+
+    script:
+    """
+    python my_script.py
+    """
+}
+```
+
+Nextflow automatically creates a uv virtual environment for each unique set of packages.
+
+Multiple packages can be specified separating them with a blank space, e.g. `numpy pandas>=2.0 scikit-learn`.
+
+The `uv` directive also accepts:
+
+- A `requirements.txt` file path: `uv '/path/to/requirements.txt'`
+- A `pyproject.toml` file path: `uv '/path/to/pyproject.toml'`
+- The path of an existing virtual environment directory
+
+See {ref}`uv-page` for more information.
+
 (process-stageinmode)=
 
 ### stageInMode

docs/uv.md

@@ -0,0 +1,130 @@
+(uv-page)=
+
+# uv environments
+
+[uv](https://docs.astral.sh/uv/) is an extremely fast Python package and project manager, written in Rust. It can install Python packages, manage virtual environments, and handle Python versions.
+
+Nextflow has built-in support for uv that allows the configuration of workflow dependencies using Python packages, requirements files, or pyproject.toml files.
+
+This allows Nextflow applications to use Python packages managed by uv, taking advantage of its speed and reliability for creating reproducible Python environments.
+
+## Prerequisites
+
+This feature requires the [uv](https://docs.astral.sh/uv/getting-started/installation/) package manager to be installed on your system.
+
+## How it works
+
+Nextflow automatically creates and activates uv virtual environments given the dependencies specified by each process.
+
+Dependencies are specified by using the {ref}`process-uv` directive, providing either the names of the required Python packages, the path of a requirements file, the path of a pyproject.toml file, or the path of an existing virtual environment directory.
+
+You can specify the directory where the uv environments are stored using the `uv.cacheDir` configuration property (see the {ref}`configuration page <config-uv>` for details). When using a computing cluster, make sure to use a shared file system path accessible from all compute nodes.
+
+:::{warning}
+The uv environment feature is not supported by executors that use remote object storage as the work directory, e.g. AWS Batch.
+:::
+
+### Enabling uv environments
+
+The use of uv packages specified using the {ref}`process-uv` directive needs to be enabled explicitly by setting the option shown below in the pipeline configuration file (i.e. `nextflow.config`):
+
+```groovy
+uv.enabled = true
+```
+
+Alternatively, it can be specified by setting the variable `NXF_UV_ENABLED=true` in your environment or by using the `-with-uv` command line option.
+
+### Use Python package names
+
+Python package names can be specified using the `uv` directive. Multiple package names can be specified by separating them with a blank space. For example:
+
+```nextflow
+process hello {
+  uv 'numpy pandas matplotlib'
+
+  script:
+  '''
+  python my_script.py
+  '''
+}
+```
+
+Using the above definition, a uv virtual environment that includes NumPy, Pandas, and Matplotlib is created and activated when the process is executed.
+
+The usual pip package syntax and naming conventions can be used. The version of a package can be specified using pip version specifiers like so: `numpy>=1.24 pandas==2.0.0`.
+
+### Use requirements files
+
+uv environments can also be defined using a requirements file. For example, given a `requirements.txt` file:
+
+```
+numpy>=1.24.0
+pandas>=2.0
+scikit-learn
+matplotlib
+```
+
+The environment for a process can be specified like so:
+
+```nextflow
+process hello {
+  uv '/path/to/requirements.txt'
+
+  script:
+  '''
+  python my_script.py
+  '''
+}
+```
+
+### Use pyproject.toml files
+
+uv can also install dependencies from a `pyproject.toml` file:
+
+```nextflow
+process hello {
+  uv '/path/to/pyproject.toml'
+
+  script:
+  '''
+  python my_script.py
+  '''
+}
+```
+
+### Use existing environments
+
+If you already have a uv virtual environment, you can use it directly by specifying the path:
+
+```nextflow
+process hello {
+  uv '/path/to/existing/venv'
+
+  script:
+  '''
+  python my_script.py
+  '''
+}
+```
+
+### Environment caching
+
+Nextflow caches uv environments so that they are created only once for each unique set of packages. The cache directory can be configured using the `uv.cacheDir` setting or the `NXF_UV_CACHEDIR` environment variable.
+
+### Python version
+
+You can specify the Python version to use when creating virtual environments:
+
+```groovy
+uv.pythonVersion = '3.12'
+```
+
+### Advanced settings
+
+The following settings are available in the `uv` scope of the Nextflow configuration:
+
+- `uv.enabled`: Enable the use of uv environments (default: `false`)
+- `uv.cacheDir`: The path where uv environments are stored
+- `uv.createTimeout`: Timeout for environment creation (default: `20 min`)
+- `uv.installOptions`: Extra command line options for `uv pip install`
+- `uv.pythonVersion`: Python version for virtual environment creation

modules/nextflow/src/main/groovy/nextflow/Session.groovy

@@ -69,6 +69,7 @@ import nextflow.script.ScriptRunner
 import nextflow.script.WorkflowMetadata
 import nextflow.script.dsl.ProcessConfigBuilder
 import nextflow.spack.SpackConfig
+import nextflow.uv.UvConfig
 import nextflow.trace.LogObserver
 import nextflow.trace.TraceObserver
 import nextflow.trace.TraceObserverFactory
@@ -1160,6 +1161,12 @@ class Session implements ISession {
         return new SpackConfig(opts, getSystemEnv())
     }
 
+    @Memoized
+    UvConfig getUvConfig() {
+        final opts = config.uv as Map ?: Collections.emptyMap()
+        return new UvConfig(opts, getSystemEnv())
+    }
+
     /**
      * Get the container engine configuration for the specified engine. If no engine is specified
      * if returns the one enabled in the configuration file. If no configuration is found

modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy

@@ -257,6 +257,12 @@ class CmdRun extends CmdBase implements HubOptions {
     @Parameter(names=['-without-spack'], description = 'Disable the use of Spack environments')
     Boolean withoutSpack
 
+    @Parameter(names=['-with-uv'], description = 'Use the specified uv environment packages or requirements file')
+    String withUv
+
+    @Parameter(names=['-without-uv'], description = 'Disable the use of uv environments')
+    Boolean withoutUv
+
     @Parameter(names=['-offline'], description = 'Do not check for remote project updates')
     boolean offline = System.getenv('NXF_OFFLINE')=='true'
 
@@ -321,6 +327,9 @@ class CmdRun extends CmdBase implements HubOptions {
         if( withSpack && withoutSpack )
             throw new AbortOperationException("Command line options `-with-spack` and `-without-spack` cannot be specified at the same time")
 
+        if( withUv && withoutUv )
+            throw new AbortOperationException("Command line options `-with-uv` and `-without-uv` cannot be specified at the same time")
+
         if( offline && latest )
             throw new AbortOperationException("Command line options `-latest` and `-offline` cannot be specified at the same time")
 

modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy

@@ -302,6 +302,10 @@ class Launcher {
                 normalized << '-'
             }
 
+            else if( current == '-with-uv' && (i==args.size() || args[i].startsWith('-'))) {
+                normalized << '-'
+            }
+
             else if( current == '-with-weblog' && (i==args.size() || args[i].startsWith('-'))) {
                 normalized << '-'
             }

modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy

@@ -625,6 +625,19 @@ class ConfigBuilder {
             config.spack.enabled = true
         }
 
+        if( cmdRun.withoutUv && config.uv instanceof Map ) {
+            // disable uv execution
+            log.debug "Disabling execution with uv as requested by command-line option `-without-uv`"
+            config.uv.enabled = false
+        }
+
+        // -- apply the uv environment
+        if( cmdRun.withUv ) {
+            if( cmdRun.withUv != '-' )
+                config.process.uv = cmdRun.withUv
+            config.uv.enabled = true
+        }
+
         // -- sets the resume option
         if( cmdRun.resume )
             config.resume = cmdRun.resume

modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy

@@ -354,6 +354,7 @@ class BashWrapperBuilder {
         binding.before_script = getBeforeScriptSnippet()
         binding.conda_activate = getCondaActivateSnippet()
         binding.spack_activate = getSpackActivateSnippet()
+        binding.uv_activate = getUvActivateSnippet()
 
         /*
          * add the task environment
@@ -573,6 +574,15 @@ class BashWrapperBuilder {
         return result
     }
 
+    private String getUvActivateSnippet() {
+        if( !uvEnv )
+            return null
+        return """\
+            # uv environment
+            source ${Escape.path(uvEnv)}/bin/activate
+            """.stripIndent()
+    }
+
     protected String getTraceCommand(String interpreter) {
         String result = "${interpreter} ${fileStr(scriptFile)}"
         if( input != null )

modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayCollector.groovy

@@ -57,6 +57,7 @@ class TaskArrayCollector {
             'containerOptions',
             // only needed when using Wave
             'conda',
+            'uv',
     ]
 
     private TaskProcessor processor

modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy

@@ -51,6 +51,8 @@ class TaskBean implements Serializable, Cloneable {
 
     Path spackEnv
 
+    Path uvEnv
+
     List<String> moduleNames
 
     Path workDir
@@ -140,6 +142,7 @@ class TaskBean implements Serializable, Cloneable {
         this.condaEnv = task.getCondaEnv()
         this.useMicromamba = task.getCondaConfig()?.useMicromamba()
         this.spackEnv = task.getSpackEnv()
+        this.uvEnv = task.getUvEnv()
         this.moduleNames = task.config.getModule()
         this.shell = task.config.getShell() ?: BashWrapperBuilder.BASH
         this.script = task.getScript()

modules/nextflow/src/main/groovy/nextflow/processor/TaskHasher.groovy

@@ -120,6 +120,12 @@ class TaskHasher {
             }
         }
 
+        // add uv packages (`uv` directive)
+        final uv = task.getUvEnv()
+        if( uv ) {
+            keys.add(uv)
+        }
+
         // add stub run marker if enabled
         if( session.stubRun && task.config.getStubBlock() ) {
             keys.add('stub-run')

modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy

@@ -51,6 +51,8 @@ import nextflow.script.params.InParam
 import nextflow.script.params.OutParam
 import nextflow.script.params.ValueOutParam
 import nextflow.spack.SpackCache
+import nextflow.uv.UvCache
+import nextflow.uv.UvConfig
 import nextflow.util.ArrayBag
 /**
  * Models a task instance
@@ -691,6 +693,29 @@ class TaskRun implements Cloneable {
         cache.getCachePathFor(config.spack as String, arch)
     }
 
+    Path getUvEnv() {
+        // note: use an explicit function instead of a closure or lambda syntax, otherwise
+        // when calling this method from a subclass it will result into a MissingMethodExeception
+        // see  https://issues.apache.org/jira/browse/GROOVY-2433
+        cache0.computeIfAbsent('uvEnv', new Function<String,Path>() {
+            @Override
+            Path apply(String it) {
+                return getUvEnv0()
+            }})
+    }
+
+    private Path getUvEnv0() {
+        if( !config.uv || !getUvConfig().isEnabled() )
+            return null
+
+        final cache = new UvCache(getUvConfig())
+        cache.getCachePathFor(config.uv as String)
+    }
+
+    UvConfig getUvConfig() {
+        return processor.session.getUvConfig()
+    }
+
     protected ContainerInfo containerInfo() {
         // note: use an explicit function instead of a closure or lambda syntax, otherwise
         // when calling this method from a subclass it will result into a MissingMethodException

modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessBuilder.groovy

@@ -81,7 +81,8 @@ class ProcessBuilder {
             'stageOutMode',
             'storeDir',
             'tag',
-            'time'
+            'time',
+            'uv'
     ]
 
     protected BaseScript ownerScript