diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8277609..dad07ef 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,18 +14,14 @@ jobs: permissions: # IMPORTANT: this permission is mandatory for Trusted Publishing id-token: write + contents: read steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 + - uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - python-version: "3.x" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel + enable-cache: true - name: Build package - run: | - python setup.py sdist bdist_wheel # Could also be python -m build + run: uv build - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + run: uv publish diff --git a/.gitignore b/.gitignore index b42c818..93d3f63 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,54 @@ +# Python *.pyc +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.tox/ + +# Distribution / packaging +*.egg +*.egg-info/ +dist/ +build/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ + +# Project specific .scripts/ logs/ .qbatch/ -.eggs/ -qbatch.egg-info/ + +# uv +uv.lock +.python-version + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7d9be97..0000000 --- a/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -dist: xenial - -language: python -python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" - - "3.7" - - "3.8" - - "3.9-dev" - # PyPy versions - - pypy - - pypy3 - -addons: - apt: - packages: - - gridengine-client - - slurm-llnl - - parallel - -install: - - export PYTHONIOENCODING=UTF8 - - pip install -r requirements-testing.txt - - pip install . - - pip install pycodestyle - -script: - - nosetests -v - - pycodestyle qbatch/qbatch.py - -deploy: - provider: pypi - user: gdevenyi - password: - secure: P4CpJPAtl6Mi0pEJIig2hwWKMgxwvBPkB6PocHU5Jd9Vn22UauCDE4Mdl1S5jDNKqo7Rz2t3QRmGfAeJDN1TgYeuaG9ZGaWDoGaqj6a8AkoPEYTwT8E7mfZYxD3WiyNd6JOzuyXPKpHNoXMznVivAI9KmxoBXoOOkX6gmUAU5Br8HWV3aG4V7NP81xLKVtn6k5i6+TY2ed+hw8PE0MpRO7bu12EOrIcR8Bc/zPF5OzE+tNonDK8pVgBAKtzg6AoS4jl6bd0/eiBsCuZGsiJkPGBBEAwJnX9DjuDvVnXu0Otj7lSB8uTE5nwaEb0XJOmgcwFo87Jct6tFhb6MSzTuexpMbqu907MR1QlIr4k0+T+8zW9te6sX+YQKp88gQlsBXEtGRkaaKHMZR0gfyAzEA8n+aLtnDQeHTc9ENfSH2wTZbHCvLa3XYTDVdB29oZSF1uf94V4vhLwAa+qM0lH2/r+KFWLrFmURX3WxMM/nAOq6EcvHKM3lHBCwY/6pUY55ROMcPCo5jU1U3/ME9IACMI1yYBksCLOV66TA3f9nNLJbevVp17zNwRniU+cU+ZtIWqmIYruT5pIFaD+Su0nw0aYRwl3DyeOxIwWv8IF2Qu+pNm8oMa7re6DR3X4YJoc9GOfWN18IxYBs+JBFN/1RNHSmME8h9yB/MD1kX5t4abA= - on: - tags: true - skip_existing: true diff --git a/CITATION.cff b/CITATION.cff index f8ffc64..770bd45 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -14,6 +14,6 @@ authors: given-names: Jon orcid: "https://orcid.org/0000-0001-6313-5701" title: qbatch -repository-code: "https://github.com/pipitone/qbatch" -version: "2.2" -date-released: 2020-03-12 +repository-code: "https://github.com/CoBrALab/qbatch" +version: "2.3.1" +date-released: 2025-08-16 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..2383c7a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,114 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +qbatch is a command-line tool and Python library for executing shell commands in parallel on high-performance computing clusters. It abstracts differences between various job schedulers (PBS/Torque, SGE, Slurm) and provides a unified interface for submitting batch jobs. + +## Common Development Commands + +### Testing +```bash +# Run all tests +pytest test/test_qbatch.py + +# Run a specific test +pytest test/test_qbatch.py::test_run_qbatch_local_piped_commands + +# Run with verbose output +pytest -v test/test_qbatch.py +``` + +### Building +```bash +# Build source distribution and wheel using uv +uv build + +# Build for local testing +uv pip install -e . +``` + +### Installation +```bash +# Install from source +pip install . + +# Install with uv +uv pip install . +``` + +## Architecture + +### Core Components + +The codebase is intentionally simple, with all logic contained in a single main file: + +- **qbatch/qbatch.py** (777 lines): Contains all functionality +- **qbatch/__init__.py**: Package exports + +### Key Functions + +**`qbatchParser(args=None)`** (line 645-772) +- Argument parser using argparse +- Parses command-line options and environment variables +- Calls `qbatchDriver()` with parsed arguments + +**`qbatchDriver(**kwargs)`** (line 341-642) +- Main driver function that orchestrates job submission +- Accepts either a command file or a `task_list` (list of command strings) +- Generates job scripts based on the selected scheduler system +- Supports "chunking" commands into groups, each running in parallel via GNU parallel + +**System-specific functions:** +- `pbs_find_jobs(patterns)` (line 238-285): Finds PBS/Torque jobs using qstat XML output +- `slurm_find_jobs(patterns)` (line 288-317): Finds Slurm jobs using squeue +- `compute_threads(ppj, ncores)` (line 228-235): Calculates threads per command + +### Templates (lines 76-155) + +The system uses template strings for generating job scheduler headers: +- `PBS_HEADER_TEMPLATE`: PBS/Torque job scripts +- `SGE_HEADER_TEMPLATE`: Grid Engine job scripts +- `SLURM_HEADER_TEMPLATE`: Slurm job scripts +- `LOCAL_TEMPLATE`: Local execution using GNU parallel +- `CONTAINER_TEMPLATE`: For containerized environments + +### Environment Variables + +All defaults can be overridden via environment variables (prefix `QBATCH_`): +- `QBATCH_SYSTEM`: Scheduler type (pbs, sge, slurm, local, container) +- `QBATCH_PPJ`: Processors per job +- `QBATCH_CHUNKSIZE`: Commands per job chunk +- `QBATCH_CORES`: Parallel commands per job +- `QBATCH_MEM`: Memory request +- `QBATCH_QUEUE`: Queue name +- `QBATCH_SCRIPT_FOLDER`: Where to write generated scripts (default: `.qbatch/`) + +### Key Concepts + +**Chunking**: Commands are divided into chunks (controlled by `-c`). Each chunk becomes one job submission. Within a job, commands run in parallel using GNU parallel (controlled by `-j`). + +**Array vs Individual Jobs**: By default, qbatch creates array jobs when chunks > 1. The `-i` flag submits individual jobs instead. + +**Job Dependencies**: The `--depend` option accepts glob patterns or job IDs to wait for before starting new jobs. + +**Environment Propagation**: Three modes (via `--env`): +- `copied`: Exports current environment variables into job script +- `batch`: Uses scheduler's native environment propagation (-V, --export=ALL) +- `none`: No environment propagation + +## Testing Notes + +Tests use `pytest` and rely on: +- Setting `QBATCH_SCRIPT_FOLDER` to a temp directory +- Testing dry-run mode (`-n`) to avoid actual job submission +- Simulating scheduler environment variables (e.g., `SGE_TASK_ID`, `PBS_ARRAYID`) + +Tests are integration-style, generating actual job scripts and verifying they produce expected output when executed. + +## Version Management + +- Version is defined in `pyproject.toml` (line 7) +- Uses `importlib.metadata` for version retrieval at runtime +- GitHub Actions workflow uses `uv` to build and publish releases to PyPI when a release is created diff --git a/README.md b/README.md index 0579c34..90b4b26 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ Execute shell command lines in parallel on Slurm, S(on) of Grid Engine (SGE), PBS/Torque clusters -[![Travis CI build status](https://travis-ci.org/pipitone/qbatch.svg?branch=master)](https://travis-ci.org/pipitone/qbatch) - qbatch is a tool for executing commands in parallel across a compute cluster. It takes as input a list of **commands** (shell command lines or executable scripts) in a file or piped to ``qbatch``. The list of commands are divided into diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..589f0c7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,75 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "qbatch" +version = "2.3.1" +description = "Execute shell command lines in parallel on Slurm, SGE and PBS/Torque clusters" +readme = "README.md" +license = { text = "Unlicense" } +requires-python = ">=3.8" +authors = [ + { name = "Jon Pipitone", email = "jon@pipitone.ca" }, + { name = "Gabriel A. Devenyi", email = "gdevenyi@gmail.com" }, +] +maintainers = [ + { name = "Jon Pipitone", email = "jon@pipitone.ca" }, + { name = "Gabriel A. Devenyi", email = "gdevenyi@gmail.com" }, +] +keywords = [ + "hpc", + "batch", + "cluster", + "slurm", + "sge", + "pbs", + "torque", + "parallel", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Science/Research", + "License :: Public Domain", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: System :: Clustering", + "Topic :: System :: Distributed Computing", + "Topic :: Utilities", +] +dependencies = [] + +[project.scripts] +qbatch = "qbatch:qbatchParser" + +[project.urls] +Homepage = "https://github.com/CoBrALab/qbatch" +Repository = "https://github.com/CoBrALab/qbatch" +Issues = "https://github.com/CoBrALab/qbatch/issues" + +[dependency-groups] +dev = [] +testing = [ + "pytest>=7.0", + "pytest-cov>=4.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["qbatch"] + +[tool.hatch.build.targets.sdist] +include = [ + "/qbatch", + "/test", + "/README.md", + "/LICENSE", + "/CLAUDE.md", +] diff --git a/qbatch/__init__.py b/qbatch/__init__.py index 26834eb..c23aea0 100644 --- a/qbatch/__init__.py +++ b/qbatch/__init__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from . import qbatch from .qbatch import qbatchParser from .qbatch import qbatchDriver diff --git a/qbatch/qbatch.py b/qbatch/qbatch.py index 4c21a6c..f5e8bed 100755 --- a/qbatch/qbatch.py +++ b/qbatch/qbatch.py @@ -1,8 +1,4 @@ #!/usr/bin/env python -from __future__ import (absolute_import, division, - print_function, unicode_literals) -from builtins import * -from future import standard_library import argparse import math import os @@ -15,28 +11,6 @@ import errno from io import open from textwrap import dedent -standard_library.install_aliases() - -# Fix python2's environment to return UTF-8 encoded items -# Stolen from https://stackoverflow.com/a/31004947/4130016 -if sys.version_info[0] < 3: - class _EnvironDict(dict): - def __getitem__(self, key): - return super(_EnvironDict, - self).__getitem__(key.encode("utf-8")).decode("utf-8") - - def __setitem__(self, key, value): - return super(_EnvironDict, self).__setitem__(key.encode("utf-8"), - value.encode("utf-8")) - - def get(self, key, failobj=None): - try: - return super(_EnvironDict, self).get(key.encode("utf-8"), - failobj).decode("utf-8") - except AttributeError: - return super(_EnvironDict, self).get(key.encode("utf-8"), - failobj) - os.environ = _EnvironDict(os.environ) def _setupVars(): diff --git a/requirements-testing.txt b/requirements-testing.txt deleted file mode 100644 index 76abfd6..0000000 --- a/requirements-testing.txt +++ /dev/null @@ -1,3 +0,0 @@ -nose>=1.0 -future -ushlex diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 2c6edea..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -future diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6fedc19..0000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[metadata] -description-file = README.md - -[bdist_wheel] -universal=1 diff --git a/setup.py b/setup.py deleted file mode 100644 index 12e2170..0000000 --- a/setup.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python - -from setuptools import setup -from io import open - -# read the contents of your README file -from os import path -this_directory = path.abspath(path.dirname(__file__)) -with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: - long_description = f.read() - -setup( - name='qbatch', - version='2.3.1', - description='Execute shell command lines in parallel on Slurm, ' - 'S(un|on of) Grid Engine (SGE) and PBS/Torque clusters', - author="Jon Pipitone, Gabriel A. Devenyi", - author_email="jon@pipitone.ca, gdevenyi@gmail.com", - license='Unlicense', - url="https://github.com/pipitone/qbatch", - long_description=long_description, - long_description_content_type='text/markdown', - entry_points={ - "console_scripts": [ - "qbatch=qbatch:qbatchParser", - ] - }, - packages=["qbatch"], - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Intended Audience :: Science/Research', - 'License :: Public Domain', - 'Natural Language :: English', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3', - 'Topic :: System :: Clustering', - 'Topic :: System :: Distributed Computing', - 'Topic :: Utilities', - ], - install_requires=[ - "future", - ], -) diff --git a/test/test_qbatch.py b/test/test_qbatch.py index 991f43a..90105b9 100644 --- a/test/test_qbatch.py +++ b/test/test_qbatch.py @@ -1,45 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from future import standard_library -from builtins import * -from builtins import str -from builtins import range import os import shutil -import sys -if sys.version_info < (3, 0): - import ushlex as shlex -else: - import shlex +import shlex from subprocess import Popen, PIPE, STDOUT import tempfile -import sys -standard_library.install_aliases() - -# Fix python2's environment to return UTF-8 encoded items -# Stolen from https://stackoverflow.com/a/31004947/4130016 -if sys.version_info[0] < 3: - class _EnvironDict(dict): - def __getitem__(self, key): - return super(_EnvironDict, - self).__getitem__(key.encode("utf-8")).decode("utf-8") - - def __setitem__(self, key, value): - return super(_EnvironDict, self).__setitem__(key.encode("utf-8"), - value.encode("utf-8")) - - def get(self, key, failobj=None): - try: - return super(_EnvironDict, self).get(key.encode("utf-8"), - failobj).decode("utf-8") - except AttributeError: - return super(_EnvironDict, self).get(key.encode("utf-8"), - failobj) - os.environ = _EnvironDict(os.environ) tempdir = None @@ -75,7 +40,7 @@ def test_qbatch_help_no_queue_binary(): If QBATCH_SYSTEM is not 'local', using --help should still work. This tests for the following issue: - https://github.com/pipitone/qbatch/issues/177 + https://github.com/CoBrALab/qbatch/issues/177 """ myenv['QBATCH_SYSTEM'] = 'slurm'