diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 0000000..4aa710c
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,36 @@
+name: "Test"
+
+on:
+  pull_request:
+  push:
+
+jobs:
+  tests:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+
+    # Use exact commit IDs instead of versions as recommended by Github
+    steps:
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
+      - uses: cachix/install-nix-action@4eae8bea4afaa8f8ea8aa638ab9a7fead2f3d21e # v31
+      - uses: cachix/cachix-action@97e5ff84d02b3b4f9936339f767e17f03c9252ed # v16
+        with:
+          name: devenv
+      - name: Install devenv.sh
+        run: nix profile add nixpkgs#devenv
+
+      # Without nix unstable, an obscure builder error is thrown
+      - name: Install nix unstable on macOS
+        if: runner.os == 'macOS'
+        run: |
+          nix profile add nixpkgs#nixVersions.latest
+          echo "$HOME/.nix-profile/bin" >> $GITHUB_PATH
+
+      # Required for the npm-format hook and node engine compatibility
+      - name: Install npm dependencies
+        run: devenv shell -- bash -c 'cd app && npm ci'
+
+      - name: Build the devenv shell and run any pre-commit hooks
+        run: devenv test
diff --git a/devenv.nix b/devenv.nix
index 1f20220..567e3e9 100644
--- a/devenv.nix
+++ b/devenv.nix
@@ -35,10 +35,15 @@
       fi
 
       cd "${config.git.root}/app"
-      npm run format
+
+      # Remove the "app/" prefix from filenames before passing to prettier, since prettier is being run in the /app directory anyways
+      npx prettier --write "''${@#app/}"
     '';
   };
 in {
+  # Disable on macos to prevent build issues
+  cachix.enable = !pkgs.stdenv.isDarwin;
+
   # https://devenv.sh/languages/
   languages = {
     javascript = {
@@ -71,7 +76,8 @@ in {
       enable = true;
       name = "npm-format";
       entry = "${lib.getExe npm-format}";
-      files = "\\.(js|ts|json|svelte|md|css|html)$";
+      # Only run prettier on files in the /app directory
+      files = "^app/.*\\.(js|ts|json|svelte|md|css|html)$";
       pass_filenames = true;
     };
   };
@@ -82,4 +88,8 @@ in {
       cwd = "${config.git.root}/app";
     };
   };
+
+  enterTest = ''
+    node --version | grep -Fx "v${pkgs.nodejs_latest.version}"
+  '';
 }