add pidlock to public functions (#45)

* add pidlock to public functions

* ruff

* ruff

* ruff

* Remove thread_lock and simplify locking

* Remove process_lock from all functions except resolve()

* fully reverting find_julia.py

* move process lock into deps.py

* use per-project lock files

* Improve lock handling in resolve() function

* More explicit version bounds

* lint

* lint

---------

Co-authored-by: Christopher Doris <github.com/cjdoris>

Author: willow-ahrens

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.7.0
+  rev: v0.9.5
   hooks:
     # Run the formatter.
     - id: ruff-format

pyproject.toml

@@ -3,7 +3,7 @@ name = "juliapkg"
 version = "0.1.15"
 description = "Julia version manager and package manager"
 authors = [{ name = "Christopher Doris" }]
-dependencies = ["semver~=3.0"]
+dependencies = ["semver >=3.0,<4.0", "filelock >=3.16,<4.0"]
 readme = "README.md"
 requires-python = ">=3.8"
 classifiers = [

src/juliapkg/compat.py

@@ -155,11 +155,11 @@ def __str__(self):
             return f"~{lo.major}.{lo.minor}.{lo.patch}"
         lostr = f"{lo.major}.{lo.minor}.{lo.patch}"
         if hi.major > 0 and hi.minor == 0 and hi.patch == 0:
-            return f"{lostr} - {hi.major-1}"
+            return f"{lostr} - {hi.major - 1}"
         if hi.minor > 0 and hi.patch == 0:
-            return f"{lostr} - {hi.major}.{hi.minor-1}"
+            return f"{lostr} - {hi.major}.{hi.minor - 1}"
         if hi.patch > 0:
-            return f"{lostr} - {hi.major}.{hi.minor}.{hi.patch-1}"
+            return f"{lostr} - {hi.major}.{hi.minor}.{hi.patch - 1}"
         raise ValueError("invalid range")
 
     def __repr__(self):

src/juliapkg/deps.py

@@ -5,6 +5,8 @@
 import sys
 from subprocess import run
 
+from filelock import FileLock
+
 from .compat import Compat, Version
 from .find_julia import find_julia, julia_version
 from .install_julia import log
@@ -287,94 +289,117 @@ def merge_any(dep, kfvs, k):
 
 
 def resolve(force=False, dry_run=False):
-    # see if we can skip resolving
-    if not force:
-        if STATE["resolved"]:
-            return True
-        deps = can_skip_resolve()
-        if deps:
-            STATE["resolved"] = True
-            STATE["executable"] = deps["executable"]
-            STATE["version"] = Version.parse(deps["version"])
-            return True
-    if dry_run:
+    # fast check to see if we have already resolved
+    if (not force) and STATE["resolved"]:
         return False
     STATE["resolved"] = False
-    # get julia compat and required packages
-    compat, pkgs = find_requirements()
-    # find a compatible julia executable
-    log(f'Locating Julia{"" if compat is None else " "+str(compat)}')
-    exe, ver = find_julia(
-        compat=compat, prefix=STATE["install"], install=True, upgrade=True
-    )
-    log(f"Using Julia {ver} at {exe}")
-    # set up the project
+    # use a lock to prevent concurrent resolution
     project = STATE["project"]
-    log(f"Using Julia project at {project}")
     os.makedirs(project, exist_ok=True)
-    if not STATE["offline"]:
-        # write a Project.toml specifying UUIDs and compatibility of required packages
-        with open(os.path.join(project, "Project.toml"), "wt") as fp:
-            print("[deps]", file=fp)
-            for pkg in pkgs:
-                print(f'{pkg.name} = "{pkg.uuid}"', file=fp)
-            print(file=fp)
-            print("[compat]", file=fp)
-            for pkg in pkgs:
-                if pkg.version:
-                    print(f'{pkg.name} = "{pkg.version}"', file=fp)
-            print(file=fp)
-        # remove Manifest.toml
-        manifest_path = os.path.join(project, "Manifest.toml")
-        if os.path.exists(manifest_path):
-            os.remove(manifest_path)
-        # install the packages
-        dev_pkgs = ", ".join([pkg.jlstr() for pkg in pkgs if pkg.dev])
-        add_pkgs = ", ".join([pkg.jlstr() for pkg in pkgs if not pkg.dev])
-        script = ["import Pkg", "Pkg.Registry.update()"]
-        if dev_pkgs:
-            script.append(f"Pkg.develop([{dev_pkgs}])")
-        if add_pkgs:
-            script.append(f"Pkg.add([{add_pkgs}])")
-        script.append("Pkg.resolve()")
-        script.append("Pkg.precompile()")
-        log("Installing packages:")
-        for line in script:
-            log("julia>", line, cont=True)
-        env = os.environ.copy()
-        if sys.executable:
-            # prefer PythonCall to use the current Python executable
-            # TODO: this is a hack, it would be better for PythonCall to detect that
-            #   Julia is being called from Python
-            env.setdefault("JULIA_PYTHONCALL_EXE", sys.executable)
-        run(
-            [exe, "--project=" + project, "--startup-file=no", "-e", "; ".join(script)],
-            check=True,
-            env=env,
+    lock_file = os.path.join(project, "lock.pid")
+    lock = FileLock(lock_file)
+    try:
+        lock.acquire(timeout=3)
+    except TimeoutError:
+        log(
+            f"Waiting for lock on {lock_file} to be freed. This normally means that"
+            " another process is resolving. If you know that no other process is"
+            " resolving, delete this file to proceed."
         )
-    # record that we resolved
-    save_meta(
-        {
-            "meta_version": META_VERSION,
-            "dev": STATE["dev"],
-            "version": str(ver),
-            "executable": exe,
-            "deps_files": {
-                filename: {
-                    "timestamp": os.path.getmtime(filename),
-                    "hash_sha256": _get_hash(filename),
-                }
-                for filename in deps_files()
-            },
-            "pkgs": [pkg.dict() for pkg in pkgs],
-            "offline": bool(STATE["offline"]),
-            "override_executable": STATE["override_executable"],
-        }
-    )
-    STATE["resolved"] = True
-    STATE["executable"] = exe
-    STATE["version"] = ver
-    return True
+        lock.acquire()
+    try:
+        # see if we can skip resolving
+        if not force:
+            deps = can_skip_resolve()
+            if deps:
+                STATE["resolved"] = True
+                STATE["executable"] = deps["executable"]
+                STATE["version"] = Version.parse(deps["version"])
+                return True
+        if dry_run:
+            return False
+        # get julia compat and required packages
+        compat, pkgs = find_requirements()
+        # find a compatible julia executable
+        log(f"Locating Julia{'' if compat is None else ' ' + str(compat)}")
+        exe, ver = find_julia(
+            compat=compat, prefix=STATE["install"], install=True, upgrade=True
+        )
+        log(f"Using Julia {ver} at {exe}")
+        # set up the project
+        log(f"Using Julia project at {project}")
+        if not STATE["offline"]:
+            # write a Project.toml specifying UUIDs and compatibility of required
+            # packages
+            with open(os.path.join(project, "Project.toml"), "wt") as fp:
+                print("[deps]", file=fp)
+                for pkg in pkgs:
+                    print(f'{pkg.name} = "{pkg.uuid}"', file=fp)
+                print(file=fp)
+                print("[compat]", file=fp)
+                for pkg in pkgs:
+                    if pkg.version:
+                        print(f'{pkg.name} = "{pkg.version}"', file=fp)
+                print(file=fp)
+            # remove Manifest.toml
+            manifest_path = os.path.join(project, "Manifest.toml")
+            if os.path.exists(manifest_path):
+                os.remove(manifest_path)
+            # install the packages
+            dev_pkgs = ", ".join([pkg.jlstr() for pkg in pkgs if pkg.dev])
+            add_pkgs = ", ".join([pkg.jlstr() for pkg in pkgs if not pkg.dev])
+            script = ["import Pkg", "Pkg.Registry.update()"]
+            if dev_pkgs:
+                script.append(f"Pkg.develop([{dev_pkgs}])")
+            if add_pkgs:
+                script.append(f"Pkg.add([{add_pkgs}])")
+            script.append("Pkg.resolve()")
+            script.append("Pkg.precompile()")
+            log("Installing packages:")
+            for line in script:
+                log("julia>", line, cont=True)
+            env = os.environ.copy()
+            if sys.executable:
+                # prefer PythonCall to use the current Python executable
+                # TODO: this is a hack, it would be better for PythonCall to detect that
+                #   Julia is being called from Python
+                env.setdefault("JULIA_PYTHONCALL_EXE", sys.executable)
+            run(
+                [
+                    exe,
+                    "--project=" + project,
+                    "--startup-file=no",
+                    "-e",
+                    "; ".join(script),
+                ],
+                check=True,
+                env=env,
+            )
+        # record that we resolved
+        save_meta(
+            {
+                "meta_version": META_VERSION,
+                "dev": STATE["dev"],
+                "version": str(ver),
+                "executable": exe,
+                "deps_files": {
+                    filename: {
+                        "timestamp": os.path.getmtime(filename),
+                        "hash_sha256": _get_hash(filename),
+                    }
+                    for filename in deps_files()
+                },
+                "pkgs": [pkg.dict() for pkg in pkgs],
+                "offline": bool(STATE["offline"]),
+                "override_executable": STATE["override_executable"],
+            }
+        )
+        STATE["resolved"] = True
+        STATE["executable"] = exe
+        STATE["version"] = ver
+        return True
+    finally:
+        lock.release()
 
 
 def executable():

src/juliapkg/install_julia.py

@@ -173,8 +173,8 @@ def download_julia(f):
             buf.write(data)
             if time.time() > t:
                 log(
-                    f"  downloaded {buf.tell()/(1<<20):.1f} MB of {size/(1<<20):.1f}"
-                    " MB",
+                    f"  downloaded {buf.tell() / (1 << 20):.1f} MB of"
+                    f" {size / (1 << 20):.1f} MB",
                     cont=True,
                 )
                 t = time.time() + freq

test/test_all.py

@@ -1,6 +1,8 @@
 import json
 import os
+import subprocess
 import tempfile
+from multiprocessing import Pool
 
 import juliapkg
 
@@ -21,6 +23,30 @@ def test_resolve():
     assert juliapkg.resolve() is True
 
 
+def resolve_in_tempdir(tempdir):
+    subprocess.run(
+        ["python", "-c", "import juliapkg; juliapkg.resolve()"],
+        env=dict(os.environ, PYTHON_JULIAPKG_PROJECT=tempdir),
+    )
+
+
+def test_resolve_contention():
+    with tempfile.TemporaryDirectory() as tempdir:
+        with open(os.path.join(tempdir, "juliapkg.json"), "w") as f:
+            f.write("""
+{
+"julia": "1",
+"packages": {
+"BenchmarkTools": {
+    "uuid": "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf",
+    "version": "1.5"
+}
+}
+}
+""")
+        Pool(5).map(resolve_in_tempdir, [tempdir] * 5)
+
+
 def test_status():
     assert juliapkg.status() is None