diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a34ed84..b494ea4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.7.0 + rev: v0.9.5 hooks: # Run the formatter. - id: ruff-format diff --git a/pyproject.toml b/pyproject.toml index fc84f14..d13bb6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "juliapkg" version = "0.1.15" description = "Julia version manager and package manager" authors = [{ name = "Christopher Doris" }] -dependencies = ["semver~=3.0"] +dependencies = ["semver >=3.0,<4.0", "filelock >=3.16,<4.0"] readme = "README.md" requires-python = ">=3.8" classifiers = [ diff --git a/src/juliapkg/compat.py b/src/juliapkg/compat.py index 4eb3bdc..973a0d7 100644 --- a/src/juliapkg/compat.py +++ b/src/juliapkg/compat.py @@ -155,11 +155,11 @@ def __str__(self): return f"~{lo.major}.{lo.minor}.{lo.patch}" lostr = f"{lo.major}.{lo.minor}.{lo.patch}" if hi.major > 0 and hi.minor == 0 and hi.patch == 0: - return f"{lostr} - {hi.major-1}" + return f"{lostr} - {hi.major - 1}" if hi.minor > 0 and hi.patch == 0: - return f"{lostr} - {hi.major}.{hi.minor-1}" + return f"{lostr} - {hi.major}.{hi.minor - 1}" if hi.patch > 0: - return f"{lostr} - {hi.major}.{hi.minor}.{hi.patch-1}" + return f"{lostr} - {hi.major}.{hi.minor}.{hi.patch - 1}" raise ValueError("invalid range") def __repr__(self): diff --git a/src/juliapkg/deps.py b/src/juliapkg/deps.py index 39ef339..b65ca19 100644 --- a/src/juliapkg/deps.py +++ b/src/juliapkg/deps.py @@ -5,6 +5,8 @@ import sys from subprocess import run +from filelock import FileLock + from .compat import Compat, Version from .find_julia import find_julia, julia_version from .install_julia import log @@ -287,94 +289,117 @@ def merge_any(dep, kfvs, k): def resolve(force=False, dry_run=False): - # see if we can skip resolving - if not force: - if STATE["resolved"]: - return True - deps = can_skip_resolve() - if deps: - STATE["resolved"] = True - STATE["executable"] = deps["executable"] - STATE["version"] = Version.parse(deps["version"]) - return True - if dry_run: + # fast check to see if we have already resolved + if (not force) and STATE["resolved"]: return False STATE["resolved"] = False - # get julia compat and required packages - compat, pkgs = find_requirements() - # find a compatible julia executable - log(f'Locating Julia{"" if compat is None else " "+str(compat)}') - exe, ver = find_julia( - compat=compat, prefix=STATE["install"], install=True, upgrade=True - ) - log(f"Using Julia {ver} at {exe}") - # set up the project + # use a lock to prevent concurrent resolution project = STATE["project"] - log(f"Using Julia project at {project}") os.makedirs(project, exist_ok=True) - if not STATE["offline"]: - # write a Project.toml specifying UUIDs and compatibility of required packages - with open(os.path.join(project, "Project.toml"), "wt") as fp: - print("[deps]", file=fp) - for pkg in pkgs: - print(f'{pkg.name} = "{pkg.uuid}"', file=fp) - print(file=fp) - print("[compat]", file=fp) - for pkg in pkgs: - if pkg.version: - print(f'{pkg.name} = "{pkg.version}"', file=fp) - print(file=fp) - # remove Manifest.toml - manifest_path = os.path.join(project, "Manifest.toml") - if os.path.exists(manifest_path): - os.remove(manifest_path) - # install the packages - dev_pkgs = ", ".join([pkg.jlstr() for pkg in pkgs if pkg.dev]) - add_pkgs = ", ".join([pkg.jlstr() for pkg in pkgs if not pkg.dev]) - script = ["import Pkg", "Pkg.Registry.update()"] - if dev_pkgs: - script.append(f"Pkg.develop([{dev_pkgs}])") - if add_pkgs: - script.append(f"Pkg.add([{add_pkgs}])") - script.append("Pkg.resolve()") - script.append("Pkg.precompile()") - log("Installing packages:") - for line in script: - log("julia>", line, cont=True) - env = os.environ.copy() - if sys.executable: - # prefer PythonCall to use the current Python executable - # TODO: this is a hack, it would be better for PythonCall to detect that - # Julia is being called from Python - env.setdefault("JULIA_PYTHONCALL_EXE", sys.executable) - run( - [exe, "--project=" + project, "--startup-file=no", "-e", "; ".join(script)], - check=True, - env=env, + lock_file = os.path.join(project, "lock.pid") + lock = FileLock(lock_file) + try: + lock.acquire(timeout=3) + except TimeoutError: + log( + f"Waiting for lock on {lock_file} to be freed. This normally means that" + " another process is resolving. If you know that no other process is" + " resolving, delete this file to proceed." ) - # record that we resolved - save_meta( - { - "meta_version": META_VERSION, - "dev": STATE["dev"], - "version": str(ver), - "executable": exe, - "deps_files": { - filename: { - "timestamp": os.path.getmtime(filename), - "hash_sha256": _get_hash(filename), - } - for filename in deps_files() - }, - "pkgs": [pkg.dict() for pkg in pkgs], - "offline": bool(STATE["offline"]), - "override_executable": STATE["override_executable"], - } - ) - STATE["resolved"] = True - STATE["executable"] = exe - STATE["version"] = ver - return True + lock.acquire() + try: + # see if we can skip resolving + if not force: + deps = can_skip_resolve() + if deps: + STATE["resolved"] = True + STATE["executable"] = deps["executable"] + STATE["version"] = Version.parse(deps["version"]) + return True + if dry_run: + return False + # get julia compat and required packages + compat, pkgs = find_requirements() + # find a compatible julia executable + log(f"Locating Julia{'' if compat is None else ' ' + str(compat)}") + exe, ver = find_julia( + compat=compat, prefix=STATE["install"], install=True, upgrade=True + ) + log(f"Using Julia {ver} at {exe}") + # set up the project + log(f"Using Julia project at {project}") + if not STATE["offline"]: + # write a Project.toml specifying UUIDs and compatibility of required + # packages + with open(os.path.join(project, "Project.toml"), "wt") as fp: + print("[deps]", file=fp) + for pkg in pkgs: + print(f'{pkg.name} = "{pkg.uuid}"', file=fp) + print(file=fp) + print("[compat]", file=fp) + for pkg in pkgs: + if pkg.version: + print(f'{pkg.name} = "{pkg.version}"', file=fp) + print(file=fp) + # remove Manifest.toml + manifest_path = os.path.join(project, "Manifest.toml") + if os.path.exists(manifest_path): + os.remove(manifest_path) + # install the packages + dev_pkgs = ", ".join([pkg.jlstr() for pkg in pkgs if pkg.dev]) + add_pkgs = ", ".join([pkg.jlstr() for pkg in pkgs if not pkg.dev]) + script = ["import Pkg", "Pkg.Registry.update()"] + if dev_pkgs: + script.append(f"Pkg.develop([{dev_pkgs}])") + if add_pkgs: + script.append(f"Pkg.add([{add_pkgs}])") + script.append("Pkg.resolve()") + script.append("Pkg.precompile()") + log("Installing packages:") + for line in script: + log("julia>", line, cont=True) + env = os.environ.copy() + if sys.executable: + # prefer PythonCall to use the current Python executable + # TODO: this is a hack, it would be better for PythonCall to detect that + # Julia is being called from Python + env.setdefault("JULIA_PYTHONCALL_EXE", sys.executable) + run( + [ + exe, + "--project=" + project, + "--startup-file=no", + "-e", + "; ".join(script), + ], + check=True, + env=env, + ) + # record that we resolved + save_meta( + { + "meta_version": META_VERSION, + "dev": STATE["dev"], + "version": str(ver), + "executable": exe, + "deps_files": { + filename: { + "timestamp": os.path.getmtime(filename), + "hash_sha256": _get_hash(filename), + } + for filename in deps_files() + }, + "pkgs": [pkg.dict() for pkg in pkgs], + "offline": bool(STATE["offline"]), + "override_executable": STATE["override_executable"], + } + ) + STATE["resolved"] = True + STATE["executable"] = exe + STATE["version"] = ver + return True + finally: + lock.release() def executable(): diff --git a/src/juliapkg/install_julia.py b/src/juliapkg/install_julia.py index 482bf99..53ebccc 100644 --- a/src/juliapkg/install_julia.py +++ b/src/juliapkg/install_julia.py @@ -173,8 +173,8 @@ def download_julia(f): buf.write(data) if time.time() > t: log( - f" downloaded {buf.tell()/(1<<20):.1f} MB of {size/(1<<20):.1f}" - " MB", + f" downloaded {buf.tell() / (1 << 20):.1f} MB of" + f" {size / (1 << 20):.1f} MB", cont=True, ) t = time.time() + freq diff --git a/test/test_all.py b/test/test_all.py index 8c93117..d2acb9a 100644 --- a/test/test_all.py +++ b/test/test_all.py @@ -1,6 +1,8 @@ import json import os +import subprocess import tempfile +from multiprocessing import Pool import juliapkg @@ -21,6 +23,30 @@ def test_resolve(): assert juliapkg.resolve() is True +def resolve_in_tempdir(tempdir): + subprocess.run( + ["python", "-c", "import juliapkg; juliapkg.resolve()"], + env=dict(os.environ, PYTHON_JULIAPKG_PROJECT=tempdir), + ) + + +def test_resolve_contention(): + with tempfile.TemporaryDirectory() as tempdir: + with open(os.path.join(tempdir, "juliapkg.json"), "w") as f: + f.write(""" +{ +"julia": "1", +"packages": { +"BenchmarkTools": { + "uuid": "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf", + "version": "1.5" +} +} +} +""") + Pool(5).map(resolve_in_tempdir, [tempdir] * 5) + + def test_status(): assert juliapkg.status() is None