Fix source distribution build and upgrade CI baselines (#83)

Summary:
Source distributions never had the right version file -- they'd always fail. `bindings/python/.../version.py` is always written out when building an sdist, so try to use it if it's available.

Add a CI baseline to try using the sdist/install with KenLM

Also move to Jammy for baseline Linux images since most Focal ones are being deprecate and upgrade GHA components depending on old Node versions. v4 of these actions breaks artifact storage with dup names/doesn't merge automatically; fix this. Contents of `dist` post-upload by wheel builder/sdist becomes
```
-rw-r--r-- 1 runner docker   60364 Feb 28 18:47 flashlight-text-0.0.6.dev336.tar.gz
-rw-r--r-- 1 runner docker 10395[8](https://github.com/flashlight/text/actions/runs/8085500728/job/22093301914#step:3:9)5 Feb 28 18:47 flashlight_text-0.0.6.dev336-cp312-cp312-macosx_10_[9](https://github.com/flashlight/text/actions/runs/8085500728/job/22093301914#step:3:10)_x86_64.whl
-rw-r--r-- 1 runner docker 1304785 Feb 28 18:47 flashlight_text-0.0.6.dev336-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
-rw-r--r-- 1 runner docker 1683721 Feb 28 18:47 flashlight_text-0.0.6.dev336-cp312-cp312-musllinux_1_1_x86_64.whl
-rw-r--r-- 1 runner docker  496932 Feb 28 18:47 flashlight_text-0.0.6.dev336-cp312-cp312-win_amd64.whl
```

### Test Plan (required)
CI + local test
```bash
pipx run build --sdist
# move tarball out of git repo where commands to fetch hashes failed
cd ${SOURCE_DIST_DIR} # after untarring
USE_KENLM=0 pip install -v .
```

### Checklist

- [x] Test coverage
- [x] Tests pass
- [x] Code formatted
- [x] Rebased on latest matter
- [x] Code documented

Pull Request resolved: #83

Reviewed By: bwasti

Differential Revision: D54318442

Pulled By: jacobkahn

fbshipit-source-id: a63d5af096178eee83049120a6248bb1bbc698a8

.circleci/config.yml

@@ -8,7 +8,7 @@ macos_env: &macos_env
 
 linux_env: &linux_env
   machine:
-    image: ubuntu-2004:202201-02
+    image: ubuntu-2204:current
 
 orbs:
   win: circleci/[email protected]
@@ -29,24 +29,39 @@ commands:
             equal: ["ON", << parameters.use_kenlm >>]
           steps:
             - run:
-                name: "Install KenLM Build Dependencies"
+                name: "Install Build Dependencies (KenLM)"
                 # TODO: remove boost once https://github.com/kpu/kenlm/pull/418 is merged
                 command: |
                   sudo apt -y install libboost-program-options-dev libboost-system-dev libboost-thread-dev \
                   libboost-test-dev liblzma-dev libbz2-dev zlib1g-dev
   install_macos_build_dependencies:
+    parameters:
+      use_kenlm:
+        type: string
     steps:
       - run:
           name: "Install Build Dependencies"
-          # TODO: remove boost once https://github.com/kpu/kenlm/pull/418 is merged
-          command: |
-            brew install cmake googletest boost zlib bzip2 lzip
+          command: brew install googletest cmake
+      - when:
+          condition:
+            equal: ["ON", << parameters.use_kenlm >>]
+          steps:
+            - run:
+                name: "Install Build Dependencies (KenLM)"
+                # TODO: remove boost once https://github.com/kpu/kenlm/pull/418 is merged
+                command: brew install boost zlib bzip2 lzip
   install_msvc_build_dependencies:
+    parameters:
+      use_kenlm:
+        type: string
     steps:
       - run:
           name: "Install Build Dependencies"
           command: |
-            choco install cmake python3 -y
+            choco install python3 cmake -y
+      - run:
+          name: "Install Build Dependencies"
+          command: choco install cmake -y
       # windows needs a path modification
       - run:
           name: "Set PATH to find CMake"
@@ -148,26 +163,33 @@ commands:
         type: string
         default: "ON"
     steps:
-      - run:
-          name: "Install Python bindings dependencies"
-          command: pip install wheel numpy cmake
       - when:
           condition:
             equal: ["ON", << parameters.use_kenlm >>]
           steps:
             - run:
                 name: "Install KenLM"
                 command: |
-                  pip install git+https://github.com/kpu/kenlm.git
+                  pip install -v git+https://github.com/kpu/kenlm.git
+      - run:
+          name: "Install Python bindings build dependencies"
+          command: |
+            pip install setuptools wheel
       - run:
           name: "Install Python Bindings"
-          command: USE_KENLM=<< parameters.use_kenlm >> pip install -v .
+          # Because of KenLM, use --no-build-isolation
+          command: |
+            USE_KENLM=<< parameters.use_kenlm >> pip install -v . --no-build-isolation
   run_python_tests:
     parameters:
       use_kenlm:
         type: string
         default: "ON"
     steps:
+      - run:
+          name: "Install test dependencies"
+          command: |
+            pip install -r bindings/python/test/requirements.txt
       - run:
           name: "Run Python Binding Tests"
           command: |
@@ -224,7 +246,7 @@ commands:
           name: Compute wheel version
           command: |
             pip install packaging
-            echo "$(python bindings/python/compute_version.py)" > BUILD_VERSION.txt
+            python bindings/python/compute_version.py > BUILD_VERSION.txt
             echo "Building version $(cat BUILD_VERSION.txt)"
       - run:
           name: Build wheels
@@ -333,7 +355,8 @@ jobs:
     shell: /bin/bash -eux -o pipefail
     steps:
       - checkout
-      - install_macos_build_dependencies
+      - install_macos_build_dependencies:
+          use_kenlm: << parameters.use_kenlm >>
       - when:
           condition:
             and:
@@ -360,7 +383,8 @@ jobs:
     shell: /bin/bash -eux -o pipefail
     steps:
       - checkout
-      - install_macos_build_dependencies
+      - install_macos_build_dependencies:
+          use_kenlm: << parameters.use_kenlm >>
       - setup_python_virtualenv:
           platform: "macos"
       - install_python_bindings:
@@ -382,7 +406,8 @@ jobs:
       size: 2xlarge
     steps:
       - checkout
-      - install_msvc_build_dependencies
+      - install_msvc_build_dependencies:
+          use_kenlm: "OFF"
       - build_flashlight_text:
           platform: "windows"
           use_kenlm: "OFF"
@@ -397,7 +422,8 @@ jobs:
       shell: bash.exe
     steps:
       - checkout
-      - install_msvc_build_dependencies
+      - install_msvc_build_dependencies:
+          use_kenlm: << parameters.use_kenlm >>
       - setup_python_virtualenv:
           platform: "windows"
       - install_python_bindings:
@@ -479,14 +505,14 @@ workflows:
     jobs:
       - ubuntu_20_gcc_9:
           build_shared_libs: "ON"
-          name: "Ubuntu 20.04 (<< matrix.resource_class >>) gcc-9 kenlm=<< matrix.use_kenlm >> standalone=<< matrix.build_standalone >>"
+          name: "Ubuntu 22.04 (<< matrix.resource_class >>) gcc-9 kenlm=<< matrix.use_kenlm >> standalone=<< matrix.build_standalone >>"
           matrix:
             parameters:
               resource_class: [xlarge, arm.xlarge] # x64, arm64
               use_kenlm: ["ON", "OFF"]
               build_standalone: ["ON", "OFF"]
       - ubuntu_20_gcc_9:
-          name: "Ubuntu 20.04 (xlarge) gcc-9 static + KenLM"
+          name: "Ubuntu 22.04 (xlarge) gcc-9 static + KenLM"
           build_shared_libs: "OFF"
           resource_class: xlarge
           use_kenlm: "ON"
@@ -502,7 +528,7 @@ workflows:
             parameters:
               resource_class: [xlarge, arm.xlarge]
               use_kenlm: ["ON", "OFF"]
-          name: "Ubuntu 20.04 (<< matrix.resource_class >>) gcc-9 Python kenlm=<< matrix.use_kenlm >>"
+          name: "Ubuntu 22.04 (<< matrix.resource_class >>) gcc-9 Python kenlm=<< matrix.use_kenlm >>"
       - macos_clang:
           matrix:
             parameters:
@@ -527,12 +553,12 @@ workflows:
   build-wheels:
     jobs:
       - linux_arm64_wheels:
-          name: "Build Python Wheels | Build wheels on ubuntu-20.04 arm64"
+          name: "Build Python Wheels | Build wheels on ubuntu-22.04 arm64"
       - macos_arm64_wheels:
           name: "Build Python Wheels | Build wheels on macOS-14 arm64"
       - upload_wheels_pypi:
           requires:
-            - "Build Python Wheels | Build wheels on ubuntu-20.04 arm64"
+            - "Build Python Wheels | Build wheels on ubuntu-22.04 arm64"
             - "Build Python Wheels | Build wheels on macOS-14 arm64"
           filters:
             branches:

.github/workflows/wheels.yml

@@ -8,10 +8,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04, windows-2022, macOS-12]
+        os: [ubuntu-22.04, windows-2022, macOS-12]
     if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'pull_request'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: actions/setup-python@v4
@@ -36,15 +36,16 @@ jobs:
           CIBW_BEFORE_BUILD: pip install -v git+https://github.com/kpu/kenlm.git
           CIBW_PRERELEASE_PYTHONS: 0
           CIBW_BUILD_VERBOSITY: 1
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           path: ./wheelhouse/*.whl
+          name: dist-${{ matrix.os }}
 
   build_sdist:
     name: Build source distribution
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -54,9 +55,27 @@ jobs:
       - name: Build sdist
         run: pipx run build --sdist
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           path: dist/*.tar.gz
+          name: dist
+
+  install_from_sdist:
+    needs: [build_sdist]
+    name: Install from source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: dist*
+          merge-multiple: true
+          path: dist
+
+      - name: Install KenLM
+        run: pip install -v git+https://github.com/kpu/kenlm.git
+
+      - name: Install from sdist (with KenLM)
+        run: ls && pip install -v dist/*.tar.gz
 
   upload_pypi:
     needs: [build_wheels, build_sdist]
@@ -66,14 +85,13 @@ jobs:
     # in the future: publish on github releases:
     # if: github.event_name == 'release' && github.event.action == 'published'
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          # unpacks default artifact into dist/
-          # name required to create an extra parent dir
-          name: artifact
+          pattern: dist*
+          merge-multiple: true
           path: dist
 
-      - uses: pypa/gh-action-pypi-publish@v1.6.4
+      - uses: pypa/gh-action-pypi-publish@v1.8.12
         with:
           verbose: true
           password: ${{ secrets.PYPI_PASSWORD }}

bindings/python/test/requirements.txt

@@ -0,0 +1 @@
+numpy

bindings/python/version.txt

@@ -1 +1 @@
-0.0.6
+0.0.7

setup.py

@@ -26,6 +26,12 @@
 PACKAGE_DIR = "bindings/python"
 ARTIFACTS_DIR = os.path.join(PACKAGE_DIR, "flashlight/lib/text")
 BUILD_VERSION_PATH = Path(os.path.join(THIS_DIR, "BUILD_VERSION.txt"))
+VERSION_TXT_PATH = Path(os.path.join(THIS_DIR, PACKAGE_DIR, "version.txt"))
+VERSION_PY_PATH = Path(
+    os.path.join(THIS_DIR, PACKAGE_DIR, "flashlight", "lib", "text", "version.py")
+)
+
+CMAKE_MINIMUM_VERSION = (3, 18)  # 3.18
 
 
 # Environment variables:
@@ -121,7 +127,7 @@ def run(self):
 
         cmake_version = re.search(r"version\s*([\d.]+)", out.decode().lower()).group(1)
         cmake_version_tuple = tuple([int(v) for v in cmake_version.split(".")])
-        if cmake_version_tuple < (3, 18):
+        if cmake_version_tuple < CMAKE_MINIMUM_VERSION:
             raise RuntimeError(
                 f"CMake >= 3.18 is required to build flashlight-text; found {cmake_version}"
             )
@@ -186,11 +192,19 @@ def main():
         version = os.getenv("BUILD_VERSION")
     elif BUILD_VERSION_PATH.is_file():
         version = BUILD_VERSION_PATH.read_text().strip()
-    else:
-        version_txt = os.path.join(THIS_DIR, PACKAGE_DIR, "version.txt")
-        with open(version_txt) as f:
+    elif VERSION_PY_PATH.is_file():
+        # see if version.py is already written (as is the case with a source distribution)
+        from bindings.python.flashlight.lib.text.version import __version__ as version
+    elif VERSION_TXT_PATH.is_file():
+        with open(VERSION_TXT_PATH) as f:
             version = f.readline().strip()
         version += get_local_version_suffix()
+    else:
+        raise RuntimeError(
+            "Could not find version.txt, BUILD_VERSION.txt, or version.py. "
+            "Please run `python bindings/python/compute_version.py` to "
+            "generate the version file."
+        )
 
     write_version_file(version)