Add organized regression suites and fix persistence, privacy, and lifecycle bugs

.github/workflows/codeql.yml

@@ -58,7 +58,7 @@ jobs:
       name: Setup .NET
       uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68
       with:
-        dotnet-version: '9.0.x'
+        dotnet-version: '10.0.x'
 
     - if: matrix.build-mode == 'manual'
       name: Build

.github/workflows/dependency-review.yml

@@ -32,5 +32,6 @@ jobs:
           # the rest are MIT/Apache-2.0). `deny-licenses` is deprecated upstream in favor
           # of this allow-list model: https://github.com/actions/dependency-review-action/issues/997
           allow-licenses: MIT, Apache-2.0, GPL-3.0-only
-          # Allow the CI-only TruffleHog GitHub Action despite its AGPL license classification.
-          allow-dependencies-licenses: pkg:githubactions/trufflesecurity/trufflehog
+          # Allow the CI-only TruffleHog GitHub Action despite its AGPL license classification,
+          # and the test-only Moq mocking library (BSD-3-Clause; tests/backend, never shipped).
+          allow-dependencies-licenses: pkg:githubactions/trufflesecurity/trufflehog, pkg:nuget/Moq

.github/workflows/regression.yml

@@ -0,0 +1,179 @@
+name: Regression tests
+
+on:
+  pull_request:
+  push:
+    branches: [main, master]
+  schedule:
+    - cron: '23 3 * * 2'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: regression-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  # Separate job: a stale inventory fails this check without skipping the tests.
+  inventory:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+      - run: python3 tests/inventory/generate.py --check
+
+  backend:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [jf10, jf12]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+      - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68
+        with:
+          dotnet-version: |
+            9.0.x
+            10.0.x
+      # Dependabot bumps package versions but cannot update the per-target lock files
+      # (packages.jf10/jf12.lock.json), so the locked restore below would stop with NU1004.
+      # Regenerate this target's lock files and fail with the fix instead.
+      - name: Check NuGet lock files (Dependabot)
+        if: github.event.pull_request.user.login == 'dependabot[bot]'
+        shell: bash
+        env:
+          TARGET: ${{ matrix.target }}
+        run: |
+          set -euo pipefail
+          for project in Jellyfin.Plugin.JellyfinEnhanced/JellyfinEnhanced.csproj tests/backend/JE.Tests.csproj; do
+            dotnet restore "$project" -p:JellyfinTarget="$TARGET" --force-evaluate
+          done
+          if ! git diff --quiet -- '*.lock.json'; then
+            git --no-pager diff -- '*.lock.json'
+            echo "::error::The $TARGET NuGet lock files are out of date after this dependency update. Regenerate both targets' lock files as described in CONTRIBUTING.md (NuGet lock files) and push them to this branch."
+            exit 1
+          fi
+      - run: python3 tests/run.py backend --target ${{ matrix.target }}
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        if: always()
+        with:
+          name: backend-${{ matrix.target }}
+          path: |
+            artifacts/regression/*.log
+            artifacts/regression/summary.json
+            artifacts/regression/backend/
+          retention-days: 14
+
+  frontend:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
+        with:
+          node-version: '22.14.0'
+          cache: npm
+      - run: npm ci
+      - run: python3 -m unittest discover -s tests/runner/tests -v
+      - run: python3 tests/run.py frontend
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        if: always()
+        with:
+          name: frontend-coverage
+          path: |
+            artifacts/regression/*.log
+            artifacts/regression/summary.json
+            artifacts/frontend-coverage/
+            coverage/
+          retention-days: 14
+
+  browser:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
+        with:
+          node-version: '22.14.0'
+          cache: npm
+      - run: npm ci
+      - run: npx playwright install --with-deps chromium firefox
+      - run: python3 tests/run.py browser
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        if: always()
+        with:
+          name: browser-results
+          path: |
+            artifacts/regression/*.log
+            artifacts/regression/summary.json
+            artifacts/browser/
+            playwright-report/
+            test-results/
+          retention-days: 14
+
+  host:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [jf10, jf12]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+      - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68
+        with:
+          dotnet-version: '10.0.x'
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
+        with:
+          node-version: '22.14.0'
+          cache: npm
+      - run: npm ci
+      - run: npx playwright install --with-deps chromium
+      - run: python3 tests/run.py host --target ${{ matrix.target }}
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        if: always()
+        with:
+          name: host-${{ matrix.target }}
+          path: |
+            artifacts/regression/*.log
+            artifacts/regression/summary.json
+            artifacts/regression/host/
+          retention-days: 14
+
+  extended:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+      - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68
+        with:
+          dotnet-version: |
+            9.0.x
+            10.0.x
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
+        with:
+          # Matches the ICU version used to generate poster language tables.
+          node-version: '26.2.0'
+          cache: npm
+      - run: npm ci
+      - run: python3 tests/run.py poster --artifacts artifacts/regression/poster-run
+      - run: python3 tests/run.py mutation --artifacts artifacts/regression/mutation-run
+        if: always()
+      - run: python3 tests/run.py history --artifacts artifacts/regression/history-run
+        if: always()
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        if: always()
+        with:
+          name: extended-results
+          path: |
+            artifacts/regression/*.log
+            artifacts/regression/summary.json
+            artifacts/regression/poster-run/
+            artifacts/regression/mutation-run/
+            artifacts/regression/history-run/
+          retention-days: 14

.github/workflows/security-scan.yml

@@ -55,20 +55,49 @@ jobs:
       - name: Setup .NET
         uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68
         with:
-          dotnet-version: '9.0.x'
+          dotnet-version: |
+            9.0.x
+            10.0.x
 
+      # Both build targets: jf12 (net10.0, the default) and jf10 (net9.0). JellyfinTarget is read
+      # from the environment as an MSBuild property. A restore or listing failure fails the step.
       - name: List vulnerable packages
-        run: dotnet list Jellyfin.Plugin.JellyfinEnhanced/JellyfinEnhanced.csproj package --vulnerable --include-transitive 2>&1 | tee vulnerable-packages.txt
+        shell: bash
+        run: |
+          set -euo pipefail
+          for target in jf12 jf10; do
+            echo "::group::$target"
+            JellyfinTarget=$target dotnet restore Jellyfin.Plugin.JellyfinEnhanced/JellyfinEnhanced.csproj
+            JellyfinTarget=$target dotnet list Jellyfin.Plugin.JellyfinEnhanced/JellyfinEnhanced.csproj package \
+              --vulnerable --include-transitive --no-restore --format json > "vulnerable-$target.json"
+            cat "vulnerable-$target.json"
+            echo "::endgroup::"
+          done
 
       - name: Check for vulnerabilities
+        shell: bash
         run: |
-          if grep -q "has the following vulnerable packages" vulnerable-packages.txt; then
-            echo "::error::Vulnerable packages found!"
-            cat vulnerable-packages.txt
-            exit 1
-          else
-            echo "No vulnerable packages found."
-          fi
+          python3 - vulnerable-jf12.json vulnerable-jf10.json <<'EOF'
+          import json, sys
+          failed = False
+          for path in sys.argv[1:]:
+              report = json.load(open(path))
+              projects = report.get('projects') or []
+              if not projects:
+                  print(f'::error::{path}: no project was scanned')
+                  failed = True
+              for problem in report.get('problems') or []:
+                  print(f"::error::{path}: {problem.get('text', problem)}")
+                  failed = True
+              for project in projects:
+                  for framework in project.get('frameworks') or []:
+                      for package in (framework.get('topLevelPackages') or []) + (framework.get('transitivePackages') or []):
+                          print(f"::error::{path}: {framework.get('framework')} {package.get('id')} {package.get('resolvedVersion')} has known vulnerabilities")
+                          failed = True
+          if failed:
+              sys.exit(1)
+          print('No vulnerable packages found for jf12 or jf10.')
+          EOF
 
   # SARIF upload for GitHub Security tab
   upload-sarif:

.gitignore

@@ -326,3 +326,11 @@ scripts/deploy*
 
 # Local test output (poster tag harnesses and captures)
 /.engineering-artifacts/
+
+# Regression infrastructure manifests are versioned; generated reports are not.
+!/package.json
+!/package-lock.json
+/artifacts/
+/coverage/
+/playwright-report/
+/test-results/

CONTRIBUTING.md

@@ -159,20 +159,24 @@ Adding a client module:
 
 ## ✅ CI Checks
 
-Every PR runs a few automated checks (GitHub Actions, `.github/workflows/`). There are no automated tests to run locally - these are all static checks:
+Pull requests run automated backend regressions for both Jellyfin targets, JavaScript behavioral tests with coverage, and browser tests, alongside the static checks below. See [Regression testing](tests/README.md) for setup, commands, reports, and current coverage gaps.
 
 | Check | What it does | Reproduce locally |
 |---|---|---|
 | **CodeQL Advanced** | Static analysis for both the C# backend and the JS frontend, looking for common security/correctness bug patterns | Not practical to run locally; check the PR's "Files changed" annotations if it flags something |
 | **Dependency Review** | Flags newly-introduced dependencies with known vulnerabilities or incompatible licenses | Only relevant if your PR changes `.csproj` package references |
 | **Security Scan** | Scans the diff for accidentally-committed secrets (API keys, tokens, credentials) with TruffleHog | `git diff` your changes yourself before pushing if you're unsure |
 | **Translation Checks** | For any locale file you touched under `js/locales/`, verifies it has valid JSON and the same key set as `en.json` (no missing/extra keys) | Diff your changed locale file's keys against `js/locales/en.json` by hand, or just keep the two in sync as you edit |
+| **NuGet lock files** | The backend regression jobs restore in locked mode from per-target lock files (`packages.jf12.lock.json`, `packages.jf10.lock.json`) beside the plugin and test projects, so a changed package reference fails with NU1004 until they are regenerated. Dependabot can't update these files, so its NuGet PRs fail a dedicated check with this fix | For each `JellyfinTarget` (`jf12`, `jf10`), run `dotnet restore Jellyfin.Plugin.JellyfinEnhanced/JellyfinEnhanced.csproj -p:JellyfinTarget=<target> --force-evaluate` and the same for `tests/backend/JE.Tests.csproj`, then commit the four lock files |
+| **Production inventory** | Part of the regression workflow: fails when `tests/docs/production-inventory.json` no longer matches the plugin. Ordinary edits and translation updates don't change it. It does change when you add, remove or rename a production file, locale, HTTP route (or its method), configuration property, scheduled task (any class implementing `IScheduledTask`) or storage literal; when a route's following attributes such as `[Authorize]` change; when a configuration property's type, default value or constructor assignment changes; or when a build target's `TargetFramework` or `JellyfinVersion` changes | Run `python3 tests/inventory/generate.py` and commit the regenerated file (`--check` only verifies it) |
 
 Two more workflows exist but aren't part of the PR gate: **Check Unused Translation Keys** and **OpenSSF Scorecard** are both maintainer-triggered/scheduled, not run against your PR - a scorecard badge or unused-key report you might see elsewhere in the repo isn't something your PR needs to pass.
 
 ## 🧪 Testing
 
-Before submitting a PR, ensure you've tested:
+After installing the prerequisites in [Regression testing](tests/README.md), run `python3 tests/run.py fast` for local feedback or `python3 tests/run.py all` for the complete suite. The full run includes disposable real Jellyfin hosts; it never uses your personal server. Automated coverage is recorded in the [coverage matrix](tests/docs/coverage-matrix.md).
+
+For behavior outside automated coverage, also verify:
 
 - [ ] Feature works as expected
 - [ ] No console errors

Jellyfin.Plugin.JellyfinEnhanced/Configuration/UserConfigurationManager.cs

@@ -608,14 +608,17 @@
 
             try
             {
-                var parsed = JsonConvert.DeserializeObject<AllReviewsStore>(json);
-                if (parsed == null)
+                // Missing/null collections and null rows are corruption, not an
+                // empty store. Do not let a subsequent write erase evidence.
+                var parsed = new AllReviewsStore { Reviews = null! };
+                JsonConvert.PopulateObject(json, parsed, new JsonSerializerSettings { CheckAdditionalContent = true });
+                if (parsed.Reviews == null || parsed.Reviews.Values.Any(review => review == null))
                 {
                     if (throwOnCorruption)
                     {
-                        _logger.Error("reviews.json deserialized to null; refusing to write over it.");
+                        _logger.Error("reviews.json contains an invalid review store; refusing to write over it.");
                         BackupCorruptFileUnlocked(filePath);
-                        throw new InvalidDataException("reviews.json deserialized to null.");
+                        throw new InvalidDataException("reviews.json contains an invalid review store.");
                     }
                     return new AllReviewsStore();
                 }
@@ -908,22 +911,55 @@
             SaveUserConfiguration(userId.ToString(), "processed-watchlist-items.json", items);
         }
 
+        /// Records a processed watchlist item for a user unless the same TMDB id and media
+        /// type is already recorded. Runs under the per-user file lock, so the scheduled
+        /// sync, a manual sync and the watchlist monitor cannot drop each other's entries.
+        /// Returns true when the item was added; a failed write is logged, not thrown, so
+        /// one user's unreadable file never stops a sync or monitor pass.
+        public bool MarkWatchlistItemProcessed(Guid userId, int tmdbId, string mediaType, string source)
+        {
+            try
+            {
+                return RmwUserConfiguration<ProcessedWatchlistItems>(userId.ToString(), "processed-watchlist-items.json", items =>
+                {
+                    if (items.Items.Any(p => p.TmdbId == tmdbId && p.MediaType == mediaType))
+                    {
+                        return 0;
+                    }
+
+                    items.Items.Add(new ProcessedWatchlistItem
+                    {
+                        TmdbId = tmdbId,
+                        MediaType = mediaType,
+                        ProcessedAt = System.DateTime.UtcNow,
+                        Source = source
+                    });
+                    return 1;
+                }) > 0;
+            }
+            catch (Exception ex)
+            {
+                _logger.Error($"Error recording processed watchlist item {mediaType}:{tmdbId} for user {userId}: {ex.Message}");
+                return false;
+            }
+        }
+
         /// Cleans up old processed watchlist items (older than specified days).
         public void CleanupOldProcessedWatchlistItems(Guid userId, int daysToKeep = 365)
         {
             try
             {
-                var items = GetProcessedWatchlistItems(userId);
                 var cutoffDate = System.DateTime.UtcNow.AddDays(-daysToKeep);
+                var removed = RmwUserConfiguration<ProcessedWatchlistItems>(userId.ToString(), "processed-watchlist-items.json", items =>
+                {
+                    var originalCount = items.Items.Count;
+                    items.Items = items.Items.Where(item => item.ProcessedAt > cutoffDate).ToList();
+                    return originalCount - items.Items.Count;
+                });
 
-                var originalCount = items.Items.Count;
-                var itemsToKeep = items.Items.Where(item => item.ProcessedAt > cutoffDate).ToList();
-
-                if (itemsToKeep.Count != originalCount)
+                if (removed > 0)
                 {
-                    items.Items = itemsToKeep;
-                    SaveProcessedWatchlistItems(userId, items);
-                    _logger.Info($"Cleaned up {originalCount - itemsToKeep.Count} old processed watchlist items for user {userId}");
+                    _logger.Info($"Cleaned up {removed} old processed watchlist items for user {userId}");
                 }
             }
             catch (Exception ex)