From 0eeb752840a12bffe0e043301731f5d1e1fcf4b1 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Thu, 17 Jul 2025 15:43:16 +1000
Subject: [PATCH 01/48] Add counter, refactor PullRequest object for beter
 names

---
 .github/workflows/push-to-dockerhub.yml      |  8 ++--
 APSIM.POStats.Collector/Program.cs           | 10 ++---
 APSIM.POStats.Portal/Controllers/Api.cs      | 29 +++++++++-----
 APSIM.POStats.Shared/Collector.cs            |  8 ++--
 APSIM.POStats.Shared/GitHub/GitHub.cs        | 12 ++++--
 APSIM.POStats.Shared/Models/PullRequest.cs   | 11 ++---
 APSIM.POStats.Shared/PullRequestFunctions.cs |  6 +--
 APSIM.POStats.Shared/StatsDbContext.cs       | 42 +++++++++++---------
 8 files changed, 74 insertions(+), 52 deletions(-)

diff --git a/.github/workflows/push-to-dockerhub.yml b/.github/workflows/push-to-dockerhub.yml
index d614c7d..1ca8186 100644
--- a/.github/workflows/push-to-dockerhub.yml
+++ b/.github/workflows/push-to-dockerhub.yml
@@ -26,7 +26,7 @@ jobs:
         id: metadata-portal
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
-          images: apsiminitiative/postats-portal2
+          images: apsiminitiative/postats2-portal
           flavor: latest=true
           tags: |
             type=ref,event=branch
@@ -38,7 +38,7 @@ jobs:
           context: .
           file: ./dockerfile
           push: true
-          target: postats-portal2
+          target: postats2-portal
           tags: ${{ steps.metadata-portal.outputs.tags }}
           labels: ${{ steps.metadata-portal.outputs.labels }}
 
@@ -46,7 +46,7 @@ jobs:
         id: metadata-collector
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
-          images: apsiminitiative/postats-collector2
+          images: apsiminitiative/postats2-collector
           flavor: latest=true
           tags: |
             type=ref,event=branch
@@ -58,6 +58,6 @@ jobs:
           context: .
           file: ./dockerfile
           push: true
-          target: postats-collector2
+          target: postats2-collector
           tags: ${{ steps.metadata-collector.outputs.tags }}
           labels: ${{ steps.metadata-collector.outputs.labels }}
diff --git a/APSIM.POStats.Collector/Program.cs b/APSIM.POStats.Collector/Program.cs
index 016cdf7..61b1ccf 100644
--- a/APSIM.POStats.Collector/Program.cs
+++ b/APSIM.POStats.Collector/Program.cs
@@ -63,7 +63,7 @@ static int Main(string[] args)
                 }
 
                 //get the Pull Request details
-                PullRequest pullRequest = Shared.Collector.RetrieveData(pullId, commitId, author, runDate, searchDirectories);
+                PullRequestDetails pullRequest = Shared.Collector.RetrieveData(pullId, commitId, author, runDate, searchDirectories);
 
                 string url = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
                 Console.WriteLine($"{url}");
@@ -75,7 +75,7 @@ static int Main(string[] args)
                 if (command == "open")
                 {
                     // Tell endpoint we're about to upload data.
-                    Task<string> response = WebUtilities.GetAsync($"{url}/open?pullrequestnumber={pullRequest.Number}&commitnumber={pullRequest.LastCommit}&author={pullRequest.Author}");
+                    Task<string> response = WebUtilities.GetAsync($"{url}/open?pullrequestnumber={pullRequest.PullRequest}&commitnumber={pullRequest.Commit}&author={pullRequest.Author}");
                     response.Wait();
                 }
                 else if (command == "upload")
@@ -87,7 +87,7 @@ static int Main(string[] args)
                 }
                 else if (command == "close")
                 {
-                    Task<string> response = WebUtilities.GetAsync($"{url}/close?pullrequestnumber={pullRequest.Number}&commitid={pullRequest.LastCommit}");
+                    Task<string> response = WebUtilities.GetAsync($"{url}/close?pullrequestnumber={pullRequest.PullRequest}&commitid={pullRequest.Commit}");
                     response.Wait();
                 }
                 //this is provided for Jenkins so it can continue to run with the original uploading code
@@ -96,7 +96,7 @@ static int Main(string[] args)
                     //get the Pull Request details
                     PullRequestJenkins pullRequestJenkins = new PullRequestJenkins();
                     pullRequestJenkins.Id = pullRequest.Id;
-                    pullRequestJenkins.Number = pullRequest.Number;
+                    pullRequestJenkins.Number = pullRequest.PullRequest;
                     pullRequestJenkins.Author = pullRequest.Author;
                     pullRequestJenkins.DateRun = pullRequest.DateRun;
                     pullRequestJenkins.DateStatsAccepted = pullRequest.DateStatsAccepted;
@@ -125,7 +125,7 @@ static int Main(string[] args)
         /// <param name="urlEnvironmentVariable"></param>
         /// <returns></returns>
         /// <exception cref="Exception"></exception>
-        private static void UploadStats(PullRequest pullRequest, string url)
+        private static void UploadStats(PullRequestDetails pullRequest, string url)
         {
             List<ApsimFile> files = new();
             files.AddRange(pullRequest.Files);
diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index eae86f2..746dc4c 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -101,11 +101,11 @@ public IActionResult Close(int pullrequestnumber, string commitid)
             }
             else
             {
-                PullRequest pr = statsDb.GetPullRequest(pullrequestnumber);
+                PullRequestDetails pr = statsDb.GetPullRequest(pullrequestnumber);
                 if (pr == null)
                     return BadRequest($"A PR with {pullrequestnumber} does not exist in the database");
                 else
-                    return BadRequest($"A PR with {pullrequestnumber} does exist, but has commit number {pr.LastCommit}, and you submitted a commit of {commitid}");
+                    return BadRequest($"A PR with {pullrequestnumber} does exist, but has commit number {pr.Commit}, and you submitted a commit of {commitid}");
             }
 
             return Ok();
@@ -116,19 +116,30 @@ public IActionResult Close(int pullrequestnumber, string commitid)
         /// <returns></returns>
         [HttpPost("adddata")]
         [RequestSizeLimit(100_000_000)]
-        public IActionResult Post([FromBody]PullRequest pullrequest)
+        public IActionResult Post([FromBody]PullRequestDetails pullrequest)
         {
             Console.WriteLine($"api/adddata called");
 
             if (pullrequest == null)
                 return BadRequest("You must supply a pull request");
 
-            if (statsDb.PullRequestWithCommitExists(pullrequest.Number, pullrequest.LastCommit))
+            if (statsDb.PullRequestWithCommitExists(pullrequest.PullRequest, pullrequest.Commit))
             {
-                Console.WriteLine($"Adding Data to PR \"{pullrequest.Number}\"");
+                Console.WriteLine($"Adding Data to PR \"{pullrequest.PullRequest}\"");
                 try
                 {
-                    statsDb.AddDataToPullRequest(pullrequest);
+                    PullRequestDetails pr = statsDb.AddDataToPullRequest(pullrequest);
+                    if (pr.CountReturned == pr.CountTotal)
+                    {
+                        string url = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
+                        Task<string> response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={pr.PullRequest}&commitId={pr.Commit}");
+                        response.Wait();
+                    }
+                    else
+                    {
+                        string message = $"Running. {pr.CountReturned} of {pr.CountTotal} completed";
+                        GitHub.SetStatus(pr.PullRequest, pr.Commit, VariableComparison.Status.Running);
+                    }
                 }
                 catch (Exception ex)
                 {
@@ -137,11 +148,11 @@ public IActionResult Post([FromBody]PullRequest pullrequest)
             }
             else
             {
-                PullRequest pr = statsDb.GetPullRequest(pullrequest.Number);
+                PullRequestDetails pr = statsDb.GetPullRequest(pullrequest.PullRequest);
                 if (pr == null)
-                    return BadRequest($"A PR with {pullrequest.Number} does not exist in the database");
+                    return BadRequest($"A PR with {pullrequest.PullRequest} does not exist in the database");
                 else
-                    return BadRequest($"A PR with {pullrequest.Number} does exist, but has commit number {pr.LastCommit}, and you submitted a commit of {pullrequest.LastCommit}");
+                    return BadRequest($"A PR with {pullrequest.PullRequest} does exist, but has commit number {pr.Commit}, and you submitted a commit of {pullrequest.Commit}");
             }
 
             return Ok();
diff --git a/APSIM.POStats.Shared/Collector.cs b/APSIM.POStats.Shared/Collector.cs
index a26baa2..31b0167 100644
--- a/APSIM.POStats.Shared/Collector.cs
+++ b/APSIM.POStats.Shared/Collector.cs
@@ -30,12 +30,12 @@ public class Collector
         /// <param name="author"></param>
         /// <param name="runDate"></param>
         /// <param name="filePaths">A collection of file paths to search.</param>
-        public static PullRequest RetrieveData(int pullId, string commitId, string author, DateTime runDate, IEnumerable<string> filePaths)
+        public static PullRequestDetails RetrieveData(int pullId, string commitId, string author, DateTime runDate, IEnumerable<string> filePaths)
         {
-            var pullRequest = new PullRequest()
+            var pullRequest = new PullRequestDetails()
             {
-                Number = pullId,
-                LastCommit = commitId,
+                PullRequest = pullId,
+                Commit = commitId,
                 Author = author,
                 DateRun = runDate,
                 Files = new List<ApsimFile>()
diff --git a/APSIM.POStats.Shared/GitHub/GitHub.cs b/APSIM.POStats.Shared/GitHub/GitHub.cs
index 3207086..6fe511f 100644
--- a/APSIM.POStats.Shared/GitHub/GitHub.cs
+++ b/APSIM.POStats.Shared/GitHub/GitHub.cs
@@ -48,7 +48,7 @@ public static GitHubPullRequestDetails GetPullRequest(int pullRequestID)
         /// </summary>
         /// <param name="pullRequestNumber">The pull request number.</param>
         /// <param name="pass">Set the status to pass?</param>
-        public static void SetStatus(int pullRequestNumber, string commitId, VariableComparison.Status status)
+        public static void SetStatus(int pullRequestNumber, string commitId, VariableComparison.Status status, string message = "")
         {
             //check we have our login token
             string token = Environment.GetEnvironmentVariable("GITHUB_TOKEN");
@@ -65,7 +65,7 @@ public static void SetStatus(int pullRequestNumber, string commitId, VariableCom
 
             //check the status of POStats for this PR
             string state = "failure";
-            string stateFormatted = status.ToString();
+           
 
             if (status == VariableComparison.Status.Same)
                 state = "success";
@@ -73,8 +73,12 @@ public static void SetStatus(int pullRequestNumber, string commitId, VariableCom
             if (status == VariableComparison.Status.Running)
                 state = "pending";
 
-            //build our check link that refers back to POStats from github
-            string serverURL = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
+            string stateFormatted = status.ToString();
+            if (String.IsNullOrEmpty(message))
+                stateFormatted = message;
+
+                //build our check link that refers back to POStats from github
+                string serverURL = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
             string urlStr = $"{serverURL}{pullRequestNumber}";
 
             //Status POST body details
diff --git a/APSIM.POStats.Shared/Models/PullRequest.cs b/APSIM.POStats.Shared/Models/PullRequest.cs
index 68ab4b7..94286c2 100644
--- a/APSIM.POStats.Shared/Models/PullRequest.cs
+++ b/APSIM.POStats.Shared/Models/PullRequest.cs
@@ -3,18 +3,19 @@
 
 namespace APSIM.POStats.Shared.Models
 {
-    public class PullRequest
+    public class PullRequestDetails
     {
         public int Id { get; set; }
-        public int Number { get; set; }
-        public string LastCommit { get; set; }
+        public int PullRequest { get; set; }
+        public string Commit { get; set; }
         public string Author { get; set; }
         public DateTime DateRun { get; set; }
         public DateTime? DateStatsAccepted { get; set; }
         public virtual List<ApsimFile> Files { get; set; }
-        public int Count { get; set; }
+        public int CountTotal { get; set; }
+        public int CountReturned { get; set; }
         public virtual List<StdOutput> Output { get; set; }
         public int? AcceptedPullRequestId { get; set; }
-        public virtual PullRequest AcceptedPullRequest { get; set; }
+        public virtual PullRequestDetails AcceptedPullRequest { get; set; }
     }
 }
\ No newline at end of file
diff --git a/APSIM.POStats.Shared/PullRequestFunctions.cs b/APSIM.POStats.Shared/PullRequestFunctions.cs
index 22fd4ec..6bcdc0c 100644
--- a/APSIM.POStats.Shared/PullRequestFunctions.cs
+++ b/APSIM.POStats.Shared/PullRequestFunctions.cs
@@ -12,7 +12,7 @@ public class PullRequestFunctions
         /// </summary>
         /// <param name="pullRequest"></param>
         /// <returns></returns>
-        public static VariableComparison.Status GetStatus(PullRequest pullRequest)
+        public static VariableComparison.Status GetStatus(PullRequestDetails pullRequest)
         {
             bool allBetterOrSame = true;
             bool allSame = true;
@@ -40,7 +40,7 @@ public static VariableComparison.Status GetStatus(PullRequest pullRequest)
 
         /// <summary>Get a list of all files for a pull request.</summary>
         /// <param name="pullRequest">The pull request.</param>
-        public static List<ApsimFileComparison> GetFileComparisons(PullRequest pullRequest)
+        public static List<ApsimFileComparison> GetFileComparisons(PullRequestDetails pullRequest)
         {
             var files = new List<ApsimFileComparison>();
             foreach (var currentFile in pullRequest.Files)
@@ -81,7 +81,7 @@ public static void RemoveSame(List<ApsimFileComparison> files)
 
         /// <summary>Update the stats in the specified pull request.</summary>
         /// <param name="pullRequest"></param>
-        public static void UpdateStats(PullRequest pullRequest)
+        public static void UpdateStats(PullRequestDetails pullRequest)
         {
             foreach (var file in pullRequest.Files)
                 foreach (var table in file.Tables)
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index fe3e9e2..a2a3e99 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -10,7 +10,7 @@ namespace APSIM.POStats.Shared
     /// </summary>
     public class StatsDbContext : DbContext
     {
-        public DbSet<PullRequest> PullRequests { get; set; }
+        public DbSet<PullRequestDetails> PullRequests { get; set; }
         public DbSet<ApsimFile> ApsimFiles { get; set; }
         public DbSet<Table> Tables { get; set; }
         public DbSet<Variable> Variables { get; set; }
@@ -39,12 +39,13 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
             var pr = GetPullRequest(pullRequestNumber);
             if (pr == null)
             {
-                pr = new PullRequest() { Number = pullRequestNumber };
+                pr = new PullRequestDetails() { PullRequest = pullRequestNumber };
                 PullRequests.Add(pr);
             }
-            pr.LastCommit = commitNumber;
+            pr.Commit = commitNumber;
             pr.Author = author;
-            pr.Count = count;
+            pr.CountTotal = count;
+            pr.CountReturned = 0;
 
             pr.Files ??= new();
             pr.Files.Clear();
@@ -61,14 +62,17 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
         /// Use case: A collector calls this method to add data to a pull request.
         /// </remarks>
         /// <param name="pullRequest">The pull request to copy the data from..</param>
-        public void AddDataToPullRequest(PullRequest fromPullRequest)
+        /// <returns>Reference to stored PullRequest</returns>
+        public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullRequest)
         {
             // Find the pull request. Should always exist if OpenPullRequest has been called.
-            var pr = PullRequests.FirstOrDefault(pr => pr.Number == fromPullRequest.Number)
-                     ?? throw new Exception($"Cannot find POStats pull request number: {fromPullRequest.Number}");
-                     
-            foreach(ApsimFile file in fromPullRequest.Files)
-                Console.WriteLine($"File \"{file.Name}\" added to PR {fromPullRequest.Number}");
+            var pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == fromPullRequest.PullRequest)
+                     ?? throw new Exception($"Cannot find POStats pull request number: {fromPullRequest.PullRequest}");
+
+            pr.CountReturned += 1;
+
+            foreach (ApsimFile file in fromPullRequest.Files)
+                Console.WriteLine($"File \"{file.Name}\" added to PR {fromPullRequest.PullRequest}");
 
             if (fromPullRequest.Files != null)
                 pr.Files.AddRange(fromPullRequest.Files);
@@ -77,6 +81,8 @@ public void AddDataToPullRequest(PullRequest fromPullRequest)
                 pr.Output.AddRange(fromPullRequest.Output);
 
             SaveChanges();
+
+            return pr;
         }
 
         /// <summary>
@@ -86,10 +92,10 @@ public void AddDataToPullRequest(PullRequest fromPullRequest)
         /// Use case: A collector calls this method to indicate that it has finished sending data to the pr.
         /// </remarks>
         /// <param name="pullRequestNumber">The pull request number.</param>
-        public PullRequest ClosePullRequest(int pullRequestNumber)
+        public PullRequestDetails ClosePullRequest(int pullRequestNumber)
         {
             // Find the pull request. Should always exist if OpenPullRequest has been called.
-            var pr = PullRequests.FirstOrDefault(pr => pr.Number == pullRequestNumber)
+            var pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == pullRequestNumber)
                      ?? throw new Exception($"Cannot find POStats pull request number: {pullRequestNumber}");
 
             // Assign the current accepted pull request.
@@ -105,7 +111,7 @@ public PullRequest ClosePullRequest(int pullRequestNumber)
         }
 
         /// <summary>Get the most recent accepted pull request.</summary>
-        public PullRequest GetMostRecentAcceptedPullRequest()
+        public PullRequestDetails GetMostRecentAcceptedPullRequest()
         {
             return PullRequests.Where(pr => pr.DateStatsAccepted != null)
                                .OrderBy(pr => pr.DateStatsAccepted)
@@ -120,10 +126,10 @@ public PullRequest GetMostRecentAcceptedPullRequest()
         public bool PullRequestWithCommitExists(int pullRequestNumber, string commitNumber)
         {
             // Try and locate the pull request
-            PullRequest pr = GetPullRequest(pullRequestNumber);
+            PullRequestDetails pr = GetPullRequest(pullRequestNumber);
             if (pr == null)
                 return false;
-            else if (pr.LastCommit == commitNumber)
+            else if (pr.Commit == commitNumber)
                 return true;
             else
                 return false;
@@ -134,10 +140,10 @@ public bool PullRequestWithCommitExists(int pullRequestNumber, string commitNumb
         /// </summary>
         /// <param name="pullRequestNumber">The pull request number.</param>
         /// <param name="commitNumber">The commit number.</param>
-        public PullRequest GetPullRequest(int pullRequestNumber)
+        public PullRequestDetails GetPullRequest(int pullRequestNumber)
         {
             // Try and locate the pull request
-            return PullRequests.FirstOrDefault(pr => pr.Number == pullRequestNumber);
+            return PullRequests.FirstOrDefault(pr => pr.PullRequest == pullRequestNumber);
         }
 
         /// <summary>
@@ -147,7 +153,7 @@ public PullRequest GetPullRequest(int pullRequestNumber)
         /// <param name="modelBuilder"></param>
         protected override void OnModelCreating(ModelBuilder modelBuilder)
         {
-            modelBuilder.Entity<PullRequest>().ToTable("PullRequest");
+            modelBuilder.Entity<PullRequestDetails>().ToTable("PullRequest");
             modelBuilder.Entity<ApsimFile>().ToTable("ApsimFile");
             modelBuilder.Entity<Table>().ToTable("Table");
             modelBuilder.Entity<Variable>().ToTable("Variable");

From abdfc983c964abb67c395ae8be1c9d871c622872 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Thu, 17 Jul 2025 15:58:35 +1000
Subject: [PATCH 02/48] Continuing to fix names

---
 APSIM.POStats.Portal/Pages/Accept.cshtml.cs   |  8 ++---
 APSIM.POStats.Portal/Pages/Heatmap.cshtml.cs  |  2 +-
 APSIM.POStats.Portal/Pages/Index.cshtml       |  8 ++---
 APSIM.POStats.Portal/Pages/Index.cshtml.cs    |  6 ++--
 .../Pages/UpdateAccepted.cshtml.cs            |  8 ++---
 APSIM.POStats.Shared/Models/ApsimFile.cs      | 32 +++++++++----------
 .../{PullRequest.cs => PullRequestDetails.cs} |  0
 APSIM.POStats.Tests/ComparisonTests.cs        | 24 +++++++-------
 APSIM.POStats.Tests/StatsDbTests.cs           | 20 ++++++------
 9 files changed, 54 insertions(+), 54 deletions(-)
 rename APSIM.POStats.Shared/Models/{PullRequest.cs => PullRequestDetails.cs} (100%)

diff --git a/APSIM.POStats.Portal/Pages/Accept.cshtml.cs b/APSIM.POStats.Portal/Pages/Accept.cshtml.cs
index 0feb817..3814846 100644
--- a/APSIM.POStats.Portal/Pages/Accept.cshtml.cs
+++ b/APSIM.POStats.Portal/Pages/Accept.cshtml.cs
@@ -15,7 +15,7 @@ public class AcceptModel : PageModel
         private readonly StatsDbContext statsDb;
 
         /// <summary>The pull request to accept stats on.</summary>
-        private PullRequest pullRequest;
+        private PullRequestDetails pullRequest;
 
         /// <summary>Constructor</summary>
         public AcceptModel(StatsDbContext stats)
@@ -27,7 +27,7 @@ public AcceptModel(StatsDbContext stats)
         public int PullRequestId => pullRequest.Id;
 
         /// <summary>The pull request .</summary>
-        public int PullRequestNumber => pullRequest.Number;
+        public int PullRequestNumber => pullRequest.PullRequest;
 
         /// <summary>Invoked when page is first loaded.</summary>
         /// <param name="id">The id of the pull request to work with.</param>
@@ -42,7 +42,7 @@ public void OnGet(int id)
         public void OnPost()
         {
             var pullRequestNumber = Convert.ToInt32(Request.Form["PullRequestNumber"]);
-            pullRequest = statsDb.PullRequests.FirstOrDefault(pr => pr.Number == pullRequestNumber);
+            pullRequest = statsDb.PullRequests.FirstOrDefault(pr => pr.PullRequest == pullRequestNumber);
             if (pullRequest == null)
                 throw new Exception($"Cannot find pull request {PullRequestNumber}");
 
@@ -58,7 +58,7 @@ public void OnPost()
                 statsDb.SaveChanges();
 
                 // Send pass/fail to gitHub
-                GitHub.SetStatus(pullRequest.Number, pullRequest.LastCommit, VariableComparison.Status.Same);
+                GitHub.SetStatus(pullRequest.PullRequest, pullRequest.Commit, VariableComparison.Status.Same);
                 Response.Redirect($"{HttpContext.Request.Scheme}://{HttpContext.Request.Host}{HttpContext.Request.PathBase.Value}/{pullRequestNumber}");
             }
         }
diff --git a/APSIM.POStats.Portal/Pages/Heatmap.cshtml.cs b/APSIM.POStats.Portal/Pages/Heatmap.cshtml.cs
index 66f70b7..0edd7d6 100644
--- a/APSIM.POStats.Portal/Pages/Heatmap.cshtml.cs
+++ b/APSIM.POStats.Portal/Pages/Heatmap.cshtml.cs
@@ -21,7 +21,7 @@ public HeatmapModel(StatsDbContext stats)
         }
 
         /// <summary>The pull request being analysed.</summary>
-        public PullRequest PullRequest { get; private set; }
+        public PullRequestDetails PullRequest { get; private set; }
 
         public string BaseUrl {  get { return $"{HttpContext.Request.Scheme}://{HttpContext.Request.Host}"; } }
 
diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml b/APSIM.POStats.Portal/Pages/Index.cshtml
index 03108be..f50dc67 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml
@@ -61,11 +61,11 @@
         <form method="post">
             <div>
                 <label for="PullRequestNumber">Number:</label>
-                <input name="PullRequestNumber" value="@Model.PullRequest.Number" readonly />
+                <input name="PullRequestNumber" value="@Model.PullRequest.PullRequest" readonly />
             </div>
             <div>
                 <label for="Commit Number">Commit:</label>
-                <input name="CommitNumber" value="@Model.PullRequest.LastCommit" readonly />
+                <input name="CommitNumber" value="@Model.PullRequest.Commit" readonly />
             </div>
             <div>
                 <label for="Author">Author:</label>
@@ -77,7 +77,7 @@
             </div>
             <div>
                 <label for="AcceptedPullRequestNumber">Accepted pull request number:</label>
-                <input name="AcceptedPullRequestNumber" value="@Model.PullRequest.AcceptedPullRequest?.Number" readonly />
+                <input name="AcceptedPullRequestNumber" value="@Model.PullRequest.AcceptedPullRequest?.PullRequest" readonly />
             </div>
             <br />
             <br />
@@ -98,7 +98,7 @@
         </form>
 
         <table class="stats-table">
-            @{
+            @{
                 var files = PullRequestFunctions.GetFileComparisons(Model.PullRequest);
                 if (Model.OnlyShowChangedStats)
                     PullRequestFunctions.RemoveSame(files);
diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml.cs b/APSIM.POStats.Portal/Pages/Index.cshtml.cs
index 0dd2a65..731a078 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml.cs
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml.cs
@@ -27,7 +27,7 @@ public IndexModel(StatsDbContext stats)
         public bool OnlyShowChangedStats { get; set; } = true;
 
         /// <summary>The pull request being analysed.</summary>
-        public PullRequest PullRequest { get; private set; }
+        public PullRequestDetails PullRequest { get; private set; }
 
         /// <summary>The Url for the web site.</summary>
         public string BaseUrl { get { return $"{HttpContext.Request.Scheme}://{HttpContext.Request.Host}"; } }
@@ -36,7 +36,7 @@ public IndexModel(StatsDbContext stats)
         /// <param name="pullRequestNumber">The pull request to work with.</param>
         public void OnGet(int pullRequestNumber)
         {
-            PullRequest = statsDb.PullRequests.FirstOrDefault(pr => pr.Number == pullRequestNumber);
+            PullRequest = statsDb.PullRequests.FirstOrDefault(pr => pr.PullRequest == pullRequestNumber);
             if (PullRequest == null)
                 throw new Exception($"Cannot find pull request #{pullRequestNumber} in stats database");
 
@@ -46,7 +46,7 @@ public void OnGet(int pullRequestNumber)
         public void OnPost()
         {
             int pullRequestNumber = Convert.ToInt32(Request.Form["PullRequestNumber"]);
-            PullRequest = statsDb.PullRequests.FirstOrDefault(pr => pr.Number == pullRequestNumber);
+            PullRequest = statsDb.PullRequests.FirstOrDefault(pr => pr.PullRequest == pullRequestNumber);
             if (PullRequest == null)
                 throw new Exception($"Cannot find pull request #{pullRequestNumber} in stats database");
 
diff --git a/APSIM.POStats.Portal/Pages/UpdateAccepted.cshtml.cs b/APSIM.POStats.Portal/Pages/UpdateAccepted.cshtml.cs
index 8a2d5e0..19ba886 100644
--- a/APSIM.POStats.Portal/Pages/UpdateAccepted.cshtml.cs
+++ b/APSIM.POStats.Portal/Pages/UpdateAccepted.cshtml.cs
@@ -15,7 +15,7 @@ public class UpdateAcceptedModel : PageModel
         private readonly StatsDbContext statsDb;
 
         /// <summary>The pull request to accept stats on.</summary>
-        private PullRequest pullRequest;
+        private PullRequestDetails pullRequest;
 
         /// <summary>Constructor</summary>
         public UpdateAcceptedModel(StatsDbContext stats)
@@ -27,7 +27,7 @@ public UpdateAcceptedModel(StatsDbContext stats)
         public int PullRequestId => pullRequest.Id;
 
         /// <summary>The pull request .</summary>
-        public int PullRequestNumber => pullRequest.Number;
+        public int PullRequestNumber => pullRequest.PullRequest;
 
         /// <summary>Invoked when page is first loaded.</summary>
         /// <param name="id">The id of the pull request to work with.</param>
@@ -42,7 +42,7 @@ public void OnGet(int id)
         public void OnPost()
         {
             var pullRequestNumber = Convert.ToInt32(Request.Form["PullRequestNumber"]);
-            pullRequest = statsDb.PullRequests.FirstOrDefault(pr => pr.Number == pullRequestNumber);
+            pullRequest = statsDb.PullRequests.FirstOrDefault(pr => pr.PullRequest == pullRequestNumber);
             if (pullRequest == null)
                 throw new Exception($"Cannot find pull request {PullRequestNumber}");
 
@@ -64,7 +64,7 @@ public void OnPost()
 
                 // Send pass/fail to gitHub
                 VariableComparison.Status status = PullRequestFunctions.GetStatus(pullRequest);
-                GitHub.SetStatus(pullRequest.Number, pullRequest.LastCommit, status);
+                GitHub.SetStatus(pullRequest.PullRequest, pullRequest.Commit, status);
                 Response.Redirect($"{HttpContext.Request.Scheme}://{HttpContext.Request.Host}{HttpContext.Request.PathBase.Value}/{pullRequestNumber}");
             }
         }
diff --git a/APSIM.POStats.Shared/Models/ApsimFile.cs b/APSIM.POStats.Shared/Models/ApsimFile.cs
index 23ceb70..298273c 100644
--- a/APSIM.POStats.Shared/Models/ApsimFile.cs
+++ b/APSIM.POStats.Shared/Models/ApsimFile.cs
@@ -1,17 +1,17 @@
-using System.Collections.Generic;
-using System.Text.Json.Serialization;
-
-namespace APSIM.POStats.Shared.Models
-{
-    public class ApsimFile
-    {
-        public int Id { get; set; }
-        public string Name { get; set; }
-        public virtual List<Table> Tables { get; set; }
-
-        [JsonIgnore]
-        public int PullRequestId { get; set; }
-        [JsonIgnore]
-        public virtual PullRequest PullRequest { get; set; }
-    }
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace APSIM.POStats.Shared.Models
+{
+    public class ApsimFile
+    {
+        public int Id { get; set; }
+        public string Name { get; set; }
+        public virtual List<Table> Tables { get; set; }
+
+        [JsonIgnore]
+        public int PullRequestId { get; set; }
+        [JsonIgnore]
+        public virtual PullRequestDetails PullRequest { get; set; }
+    }
 }
\ No newline at end of file
diff --git a/APSIM.POStats.Shared/Models/PullRequest.cs b/APSIM.POStats.Shared/Models/PullRequestDetails.cs
similarity index 100%
rename from APSIM.POStats.Shared/Models/PullRequest.cs
rename to APSIM.POStats.Shared/Models/PullRequestDetails.cs
diff --git a/APSIM.POStats.Tests/ComparisonTests.cs b/APSIM.POStats.Tests/ComparisonTests.cs
index da0ee95..03c1f1e 100644
--- a/APSIM.POStats.Tests/ComparisonTests.cs
+++ b/APSIM.POStats.Tests/ComparisonTests.cs
@@ -13,7 +13,7 @@ public class ComparisonTests
         [Test]
         public void FileStatusWorks()
         {
-            var acceptedPullRequest = new PullRequest()
+            var acceptedPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
@@ -21,7 +21,7 @@ public void FileStatusWorks()
                     new ApsimFile() { Name = "file3" }
                 }
             };
-            var currentPullRequest = new PullRequest()
+            var currentPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
@@ -43,7 +43,7 @@ public void FileStatusWorks()
         [Test]
         public void TableStatusWorks()
         {
-            var acceptedPullRequest = new PullRequest()
+            var acceptedPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
@@ -58,7 +58,7 @@ public void TableStatusWorks()
                     }
                 }
             };
-            var currentPullRequest = new PullRequest()
+            var currentPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
@@ -88,7 +88,7 @@ public void TableStatusWorks()
         [Test]
         public void TableStatusWorksWithNoAccepted()
         {
-            var currentPullRequest = new PullRequest()
+            var currentPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
@@ -116,7 +116,7 @@ public void TableStatusWorksWithNoAccepted()
         [Test]
         public void VariableStatusWork()
         {
-            var acceptedPullRequest = new PullRequest()
+            var acceptedPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
@@ -152,7 +152,7 @@ public void VariableStatusWork()
                     }
                 }
             };
-            var currentPullRequest = new PullRequest()
+            var currentPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
@@ -214,7 +214,7 @@ public void VariableStatusWork()
         [Test]
         public void VariableStatusWorksWithNoAccepted()
         {
-            var currentPullRequest = new PullRequest()
+            var currentPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
@@ -383,7 +383,7 @@ public void VariableComparisonOfInfinityPasses()
         [Test]
         public void EnsureOnlyShowChangedStatsWorks()
         {
-            var acceptedPullRequest = new PullRequest()
+            var acceptedPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
@@ -419,7 +419,7 @@ public void EnsureOnlyShowChangedStatsWorks()
                     }
                 }
             };
-            var currentPullRequest = new PullRequest()
+            var currentPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
@@ -482,7 +482,7 @@ public void EnsureOnlyShowChangedStatsWorks()
         [Test]
         public void MissingTableDetected()
         {
-            var acceptedPullRequest = new PullRequest()
+            var acceptedPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
@@ -526,7 +526,7 @@ public void MissingTableDetected()
                     }
                 }
             };
-            var currentPullRequest = new PullRequest()
+            var currentPullRequest = new PullRequestDetails()
             {
                 Files = new List<ApsimFile>()
                 {
diff --git a/APSIM.POStats.Tests/StatsDbTests.cs b/APSIM.POStats.Tests/StatsDbTests.cs
index 710716b..fbc8136 100644
--- a/APSIM.POStats.Tests/StatsDbTests.cs
+++ b/APSIM.POStats.Tests/StatsDbTests.cs
@@ -17,7 +17,7 @@ public void TestOpenPullRequestThatAlreadyExists()
         db.OpenPullRequest(1234, "1", "author", 0);
 
         // Make sure the pr exists and has no data.
-        var pr = db.PullRequests.First(pr => pr.Number == 1234);
+        var pr = db.PullRequests.First(pr => pr.PullRequest == 1234);
         Assert.That(pr, !Is.Null);
         Assert.That(pr.Files.Count, Is.EqualTo(0));
     }
@@ -32,7 +32,7 @@ public void TestOpenPullRequestThatDoesntAlreadyExist()
         db.OpenPullRequest(5678, "1", "author", 0);
 
         // Make sure the pr exists and has no ydata.
-        var pr = db.PullRequests.First(pr => pr.Number == 5678);
+        var pr = db.PullRequests.First(pr => pr.PullRequest == 5678);
         Assert.That(pr, !Is.Null);
         Assert.That(pr.Files.Count, Is.EqualTo(0));
     }
@@ -45,9 +45,9 @@ public void TestAddDataToPullRequest()
         using var db = CreateInMemoryDB("db3");
 
         db.OpenPullRequest(1234, "1", "author", 0);
-        PullRequest prToAdd = new()
+        PullRequestDetails prToAdd = new()
         {
-            Number = 1234,
+            PullRequest = 1234,
             Files = new()
             {
                 new ApsimFile()
@@ -85,10 +85,10 @@ public void TestAddDataToPullRequest()
         db.ClosePullRequest(1234);
 
         // Make sure the pr exists and has no data.
-        var pr = db.PullRequests.First(pr => pr.Number == 1234);
+        var pr = db.PullRequests.First(pr => pr.PullRequest == 1234);
         Assert.That(pr, !Is.Null);
         Assert.That(pr.DateStatsAccepted, Is.Null); // stats not accepted
-        Assert.That(pr.AcceptedPullRequest.Number, Is.EqualTo(5678));     // most recent pr that has been accepted.
+        Assert.That(pr.AcceptedPullRequest.PullRequest, Is.EqualTo(5678));     // most recent pr that has been accepted.
         Assert.That(pr.Files.Count, Is.EqualTo(1));
         var variable = pr.Files[0].Tables[0].Variables[0];
         Assert.That(variable.Name, Is.EqualTo("B"));
@@ -115,10 +115,10 @@ private StatsDbContext CreateInMemoryDB(string databaseName)
             .UseInMemoryDatabase(databaseName)
             .Options;
         var db = new StatsDbContext(options);
-        db.PullRequests.Add(new PullRequest
+        db.PullRequests.Add(new PullRequestDetails
         {
             Id = 1,
-            Number = 1234,
+            PullRequest = 1234,
             DateStatsAccepted = new DateTime(2021,1,1),
             Files = new()
             {
@@ -153,10 +153,10 @@ private StatsDbContext CreateInMemoryDB(string databaseName)
                 }
             }
         });
-        db.PullRequests.Add(new PullRequest
+        db.PullRequests.Add(new PullRequestDetails
         {
             Id = 2,
-            Number = 5678,
+            PullRequest = 5678,
             DateStatsAccepted = new DateTime(2020,1,1),
             Files = new()
             {

From 8c9882b1f46d95c75a64832c8396d35cb4a62616 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Fri, 18 Jul 2025 10:57:23 +1000
Subject: [PATCH 03/48] Counter responding to github

---
 APSIM.POStats.Portal/Controllers/Api.cs |  3 ++-
 APSIM.POStats.Shared/GitHub/GitHub.cs   | 10 +++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index 746dc4c..46c0755 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -129,6 +129,7 @@ public IActionResult Post([FromBody]PullRequestDetails pullrequest)
                 try
                 {
                     PullRequestDetails pr = statsDb.AddDataToPullRequest(pullrequest);
+                    Console.WriteLine($"{pr.CountReturned} of {pr.CountTotal} completed.");
                     if (pr.CountReturned == pr.CountTotal)
                     {
                         string url = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
@@ -138,7 +139,7 @@ public IActionResult Post([FromBody]PullRequestDetails pullrequest)
                     else
                     {
                         string message = $"Running. {pr.CountReturned} of {pr.CountTotal} completed";
-                        GitHub.SetStatus(pr.PullRequest, pr.Commit, VariableComparison.Status.Running);
+                        GitHub.SetStatus(pr.PullRequest, pr.Commit, VariableComparison.Status.Running, message);
                     }
                 }
                 catch (Exception ex)
diff --git a/APSIM.POStats.Shared/GitHub/GitHub.cs b/APSIM.POStats.Shared/GitHub/GitHub.cs
index 6fe511f..574523d 100644
--- a/APSIM.POStats.Shared/GitHub/GitHub.cs
+++ b/APSIM.POStats.Shared/GitHub/GitHub.cs
@@ -37,8 +37,6 @@ public static GitHubPullRequestDetails GetPullRequest(int pullRequestID)
             string state = dictionary["state"].ToString();
             string statusURL = dictionary["statuses_url"].ToString();
 
-            Console.WriteLine($"Github Status Update: {number} {author} {dateTime} {state} {statusURL}");
-
             //return the result as a GitHubPullRequestDetails because we don't need all the data the base class has
             return new GitHubPullRequestDetails(number, author, dateTime, state, statusURL);
         }
@@ -74,13 +72,15 @@ public static void SetStatus(int pullRequestNumber, string commitId, VariableCom
                 state = "pending";
 
             string stateFormatted = status.ToString();
-            if (String.IsNullOrEmpty(message))
+            if (!String.IsNullOrEmpty(message))
                 stateFormatted = message;
 
-                //build our check link that refers back to POStats from github
-                string serverURL = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
+            //build our check link that refers back to POStats from github
+            string serverURL = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
             string urlStr = $"{serverURL}{pullRequestNumber}";
 
+            Console.WriteLine($"Github Status Update: {state} {urlStr} {stateFormatted} {"APSIM.POStats2"}");
+
             //Status POST body details
             GitHubStatusDetails body = new GitHubStatusDetails(state, urlStr, stateFormatted, "APSIM.POStats2");
 

From 6e5d9024d7c74f311e9bf930f9e2f44ebf8a66d8 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Fri, 18 Jul 2025 13:22:48 +1000
Subject: [PATCH 04/48] Working Counter

---
 APSIM.POStats.Collector/Program.cs            | 24 ++++++++++---------
 APSIM.POStats.Portal/Controllers/Api.cs       |  6 ++---
 APSIM.POStats.Shared/Collector.cs             | 12 ++++++++++
 .../Models/PullRequestDetails.cs              |  2 +-
 APSIM.POStats.Shared/StatsDbContext.cs        |  3 +--
 5 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/APSIM.POStats.Collector/Program.cs b/APSIM.POStats.Collector/Program.cs
index 61b1ccf..495216b 100644
--- a/APSIM.POStats.Collector/Program.cs
+++ b/APSIM.POStats.Collector/Program.cs
@@ -129,22 +129,24 @@ private static void UploadStats(PullRequestDetails pullRequest, string url)
         {
             List<ApsimFile> files = new();
             files.AddRange(pullRequest.Files);
-            foreach (var file in files)
-            {
-                // Upload data for one file only.
-                pullRequest.Files.Clear();
-                pullRequest.Files.Add(file);
 
-                try
+            //In the case we have no files produced, just send it back empty.
+            if (files.Count == 0)
+            {
+                Task<string> response = WebUtilities.PostAsync($"{url}/adddata", pullRequest, null);
+                response.Wait();
+            }
+            else
+            {
+                foreach (var file in files)
                 {
+                    // Upload data for one file only.
+                    pullRequest.Files = new List<ApsimFile>();
+                    pullRequest.Files.Add(file);
+
                     Task<string> response = WebUtilities.PostAsync($"{url}/adddata", pullRequest, null);
                     response.Wait();
                 }
-                catch (Exception exception)
-                {
-                    Console.WriteLine($"Error when collecting file {file.Name}");
-                    Console.WriteLine(exception.Message);
-                }
             }
         }
         
diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index 46c0755..c693bec 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -50,12 +50,12 @@ public IActionResult Open(int pullrequestnumber, string commitid, int count, str
                 return BadRequest("You must supply an author");
             try
             {
-                GitHub.SetStatus(pullrequestnumber, commitid, VariableComparison.Status.Running);
+                GitHub.SetStatus(pullrequestnumber, commitid, VariableComparison.Status.Running, $"Running: 0 of {count} completed");
 
                 statsDb.OpenPullRequest(pullrequestnumber, commitid, author, count);
 
                 // Create a timer to close the PR after 30 minutes
-                TimeoutTimer timeoutTimer = new TimeoutTimer {Interval=1800000, PullRequestNumber=pullrequestnumber, CommitId=commitid};
+                TimeoutTimer timeoutTimer = new TimeoutTimer {Interval=3600000, PullRequestNumber=pullrequestnumber, CommitId=commitid};
                 timeoutTimer.Elapsed += OnTimeout;
                 timeoutTimer.AutoReset = false;
                 timeoutTimer.Start();
@@ -138,7 +138,7 @@ public IActionResult Post([FromBody]PullRequestDetails pullrequest)
                     }
                     else
                     {
-                        string message = $"Running. {pr.CountReturned} of {pr.CountTotal} completed";
+                        string message = $"Running: {pr.CountReturned} of {pr.CountTotal} completed";
                         GitHub.SetStatus(pr.PullRequest, pr.Commit, VariableComparison.Status.Running, message);
                     }
                 }
diff --git a/APSIM.POStats.Shared/Collector.cs b/APSIM.POStats.Shared/Collector.cs
index 31b0167..e6d766b 100644
--- a/APSIM.POStats.Shared/Collector.cs
+++ b/APSIM.POStats.Shared/Collector.cs
@@ -67,6 +67,18 @@ public static PullRequestDetails RetrieveData(int pullId, string commitId, strin
                         errorMessages += ex.ToString();
                     }
                 }
+                pullRequest.Output = "";
+                foreach (FileInfo fileInfo in info.GetFiles("stdout.txt", SearchOption.AllDirectories))
+                {
+                    using (StreamReader sr = fileInfo.OpenText())
+                    {
+                        string text = "";
+                        while ((text = sr.ReadLine()) != null)
+                        {
+                            pullRequest.Output += text + "\n";
+                        }
+                    }
+                }
             }
             if (errorMessages.Length > 0)
                 throw new Exception(errorMessages);
diff --git a/APSIM.POStats.Shared/Models/PullRequestDetails.cs b/APSIM.POStats.Shared/Models/PullRequestDetails.cs
index 94286c2..d51638b 100644
--- a/APSIM.POStats.Shared/Models/PullRequestDetails.cs
+++ b/APSIM.POStats.Shared/Models/PullRequestDetails.cs
@@ -14,7 +14,7 @@ public class PullRequestDetails
         public virtual List<ApsimFile> Files { get; set; }
         public int CountTotal { get; set; }
         public int CountReturned { get; set; }
-        public virtual List<StdOutput> Output { get; set; }
+        public string Output { get; set; }
         public int? AcceptedPullRequestId { get; set; }
         public virtual PullRequestDetails AcceptedPullRequest { get; set; }
     }
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index a2a3e99..62851c5 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -77,8 +77,7 @@ public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullReques
             if (fromPullRequest.Files != null)
                 pr.Files.AddRange(fromPullRequest.Files);
 
-            if (fromPullRequest.Output != null)
-                pr.Output.AddRange(fromPullRequest.Output);
+            pr.Output += fromPullRequest.Output;
 
             SaveChanges();
 

From 7d1b324b5b83b79811103f4be66097e6fba6bf91 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Fri, 18 Jul 2025 15:32:23 +1000
Subject: [PATCH 05/48] Logs appearing on page

---
 APSIM.POStats.Portal/Pages/Index.cshtml      | 28 +++++++++++++++++++-
 APSIM.POStats.Portal/Pages/Index.cshtml.cs   |  6 +++++
 APSIM.POStats.Shared/Collector.cs            | 27 ++++++++++++-------
 APSIM.POStats.Shared/PullRequestFunctions.cs |  2 +-
 APSIM.POStats.Shared/StatsDbContext.cs       |  1 +
 5 files changed, 52 insertions(+), 12 deletions(-)

diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml b/APSIM.POStats.Portal/Pages/Index.cshtml
index f50dc67..f5efbe7 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml
@@ -60,7 +60,7 @@
 
         <form method="post">
             <div>
-                <label for="PullRequestNumber">Number:</label>
+                <label for="PullRequestNumber">Pull Request:</label>
                 <input name="PullRequestNumber" value="@Model.PullRequest.PullRequest" readonly />
             </div>
             <div>
@@ -93,6 +93,18 @@
                     <button name="toggle-button" type="submit">Show changed stats</button>
                 }
             </div>
+            <div>
+                @if (Model.ShowLogs)
+                {
+                    <input type="hidden" name="LogsLabel" value="Hide Output Logs" readonly />
+                    <button name="toggle-button-logs" type="submit">Hide Output Logs</button>
+                }
+                else
+                {
+                    <input type="hidden" name="LogsLabel" value="Show Output Logs" readonly />
+                    <button name="toggle-button-logs" type="submit">Show Output Logs</button>
+                }
+            </div>
             <br />
             <br />
         </form>
@@ -157,5 +169,19 @@
                 }
             }
         </table>
+        <div class="output-logs">
+            logs:
+            @if (Model.ShowLogs) 
+            {
+                @foreach (string line in @Model.PullRequest.Output.Split("\n"))
+                {
+                    @Html.Encode(line)<br />
+                }
+            }
+            else
+            {
+                @Html.Raw("Hidden")
+            }
+        </div>
     </div>
 </div>
diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml.cs b/APSIM.POStats.Portal/Pages/Index.cshtml.cs
index 731a078..670de26 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml.cs
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml.cs
@@ -26,6 +26,9 @@ public IndexModel(StatsDbContext stats)
         /// <summary>Only show changed stats?</summary>
         public bool OnlyShowChangedStats { get; set; } = true;
 
+        /// <summary>Show standard output logs</summary>
+        public bool ShowLogs { get; set; } = true;
+
         /// <summary>The pull request being analysed.</summary>
         public PullRequestDetails PullRequest { get; private set; }
 
@@ -52,6 +55,9 @@ public void OnPost()
 
             var statsLabel = Request.Form["StatsLabel"].ToString();
             OnlyShowChangedStats = statsLabel == "Showing all stats.";
+
+            var logsLabel = Request.Form["LogsLabel"].ToString();
+            ShowLogs = logsLabel == "Show Output Logs";
         }
 
         /// <summary>Emit html to display tick/cross.</summary>
diff --git a/APSIM.POStats.Shared/Collector.cs b/APSIM.POStats.Shared/Collector.cs
index e6d766b..e8f189f 100644
--- a/APSIM.POStats.Shared/Collector.cs
+++ b/APSIM.POStats.Shared/Collector.cs
@@ -42,6 +42,7 @@ public static PullRequestDetails RetrieveData(int pullId, string commitId, strin
             };
 
             string errorMessages = string.Empty;
+            pullRequest.Output = "";
             foreach (var filePath in filePaths)
             {
                 string currentPath = filePath.Trim();
@@ -64,24 +65,30 @@ public static PullRequestDetails RetrieveData(int pullId, string commitId, strin
                     }
                     catch (Exception ex)
                     {
-                        errorMessages += ex.ToString();
+                        errorMessages += ex.ToString() + "\n";
                     }
                 }
-                pullRequest.Output = "";
-                foreach (FileInfo fileInfo in info.GetFiles("stdout.txt", SearchOption.AllDirectories))
+            }
+
+            FileInfo[] files = new DirectoryInfo("./").GetFiles("*stdout.txt", SearchOption.AllDirectories);
+            foreach (FileInfo fileInfo in files)
+            {
+                Console.WriteLine(fileInfo.FullName);
+                using (StreamReader sr = fileInfo.OpenText())
                 {
-                    using (StreamReader sr = fileInfo.OpenText())
+                    string text = "";
+                    while ((text = sr.ReadLine()) != null)
                     {
-                        string text = "";
-                        while ((text = sr.ReadLine()) != null)
-                        {
-                            pullRequest.Output += text + "\n";
-                        }
+                        pullRequest.Output += text + "\n";
                     }
                 }
             }
+
             if (errorMessages.Length > 0)
-                throw new Exception(errorMessages);
+            {
+                pullRequest.Output += errorMessages;
+                pullRequest.Files.Clear();
+            }
 
             return pullRequest;
         }
diff --git a/APSIM.POStats.Shared/PullRequestFunctions.cs b/APSIM.POStats.Shared/PullRequestFunctions.cs
index 6bcdc0c..68bdbf9 100644
--- a/APSIM.POStats.Shared/PullRequestFunctions.cs
+++ b/APSIM.POStats.Shared/PullRequestFunctions.cs
@@ -31,7 +31,7 @@ public static VariableComparison.Status GetStatus(PullRequestDetails pullRequest
                 }
             }
             if (allSame)
-                return VariableComparison.Status.Same; 
+                return VariableComparison.Status.Same;
             else if (allBetterOrSame)
                 return VariableComparison.Status.Better;
             else
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 62851c5..212ba93 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -46,6 +46,7 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
             pr.Author = author;
             pr.CountTotal = count;
             pr.CountReturned = 0;
+            pr.Output = "";
 
             pr.Files ??= new();
             pr.Files.Clear();

From 7808e0c8d4894539a0fc8d2e2337f4de1ec2eba1 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Tue, 5 Aug 2025 09:35:00 +1000
Subject: [PATCH 06/48] Updates to chart page

---
 APSIM.POStats.Portal/Pages/Chart.cshtml       | 148 +++++++++---------
 APSIM.POStats.Portal/Pages/Chart.cshtml.cs    |  12 +-
 APSIM.POStats.Portal/Pages/Index.cshtml       | 125 ++++++++-------
 APSIM.POStats.Portal/Pages/Index.cshtml.cs    |  47 +++++-
 .../Pages/Shared/_Layout.cshtml               | 100 ++++++------
 5 files changed, 242 insertions(+), 190 deletions(-)

diff --git a/APSIM.POStats.Portal/Pages/Chart.cshtml b/APSIM.POStats.Portal/Pages/Chart.cshtml
index 31cd530..dfa247a 100644
--- a/APSIM.POStats.Portal/Pages/Chart.cshtml
+++ b/APSIM.POStats.Portal/Pages/Chart.cshtml
@@ -1,75 +1,75 @@
-@page
-@model ChartModel
-@{
-    ViewData["Title"] = "Chart";
-}
-@using APSIM.POStats.Shared;
-@{
-    <div>
-        <div class="container">
-            <div id="chart" class="float-left"></div>
-            <div id="stats" class="float-right">
-                <table>
-                    <tr>
-                        <td class="stats-cell">"N"</td>
-                        <td class="stats-cell">@Html.Raw(IndexModel.EmitNumber(@Model.Variable.CurrentN, isAccepted: false))</td>
-                        <td class="stats-cell">@Html.Raw(IndexModel.EmitNumber(@Model.Variable.AcceptedN, isAccepted: true))</td>
-                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(@Model.Variable.NStatus))</td>
-                        <td class="stats-cell">&nbsp;&nbsp;</td>
-                    </tr>
-                    <tr>
-                        <td class="stats-cell">"RMSE"</td>
-                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.CurrentRMSE, 0, 6, isAccepted: false))</td>
-                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.AcceptedRMSE, 0, 6, isAccepted: true))</td>
-                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(@Model.Variable.RMSEStatus))</td>
-                        <td class="stats-cell">&nbsp;&nbsp;</td>
-                    </tr>
-                    <tr>
-                        <td class="stats-cell">"NSE"</td>
-                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.CurrentNSE, 3, 0, isAccepted: false))</td>
-                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.AcceptedNSE, 3, 0, isAccepted: true))</td>
-                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(@Model.Variable.NSEStatus))@VariableFunctions.NSERating(@Model.Variable.CurrentNSE)</td>
-                        <td class="stats-cell">&nbsp;&nbsp;</td>
-                    </tr>
-                    <tr>
-                        <td class="stats-cell">"RSR"</td>
-                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.CurrentRSR, 3, 0, isAccepted: false))</td>
-                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.AcceptedRSR, 3, 0, isAccepted: true))</td>
-                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(@Model.Variable.RSRStatus))@VariableFunctions.RSRRating(@Model.Variable.CurrentNSE)</td>
-                    </tr>
-                </table>
-            </div>
-        </div>
-    </div>
-    <script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script>
-
-    <script type="text/javascript">
-
-        google.charts.load('current', { 'packages': ['corechart'] });
-        google.charts.setOnLoadCallback(drawChart);
-        function drawChart() {
-            var jsonData = $.ajax({
-                url: location.pathname + "?handler=ChartData&id=@Model.Id",
-                dataType: "json",
-                async: false
-            }).responseText;
-            
-            var data = new google.visualization.DataTable(jsonData);
-            var options = {
-                'title': '@Model.Name',
-                'width': 600,
-                'height': 600,
-                hAxis: { title: 'Observed'/*, minValue: 0, maxValue: 800*/ },
-                vAxis: { title: 'Predicted'/*, minValue: 0, maxValue: 800*/ },
-                legend: { position: 'right' },
-                trendlines: { 0: {} },    // Draw a trendline for data series 0.
-                series: {
-                    2: { lineWidth: 1, pointSize: 0 } // Series 2 (1:1) is a line.
-                }
-            };
-
-            var chart = new google.visualization.ScatterChart(document.getElementById('chart'));
-            chart.draw(data, options);
-        }
-    </script>
+@page
+@model ChartModel
+@{
+    ViewData["Title"] = "Chart";
+}
+@using APSIM.POStats.Shared;
+@{
+    <div>
+        <div class="container">
+            <div id="chart" class="float-left"></div>
+            <div id="stats" class="float-right">
+                <table>
+                    <tr>
+                        <td class="stats-cell">"N"</td>
+                        <td class="stats-cell">@Html.Raw(IndexModel.EmitNumber(@Model.Variable.CurrentN, isAccepted: false))</td>
+                        <td class="stats-cell">@Html.Raw(IndexModel.EmitNumber(@Model.Variable.AcceptedN, isAccepted: true))</td>
+                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(@Model.Variable.NStatus))</td>
+                        <td class="stats-cell">&nbsp;&nbsp;</td>
+                    </tr>
+                    <tr>
+                        <td class="stats-cell">"RMSE"</td>
+                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.CurrentRMSE, 0, 6, isAccepted: false))</td>
+                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.AcceptedRMSE, 0, 6, isAccepted: true))</td>
+                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(@Model.Variable.RMSEStatus))</td>
+                        <td class="stats-cell">&nbsp;&nbsp;</td>
+                    </tr>
+                    <tr>
+                        <td class="stats-cell">"NSE"</td>
+                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.CurrentNSE, 3, 0, isAccepted: false))</td>
+                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.AcceptedNSE, 3, 0, isAccepted: true))</td>
+                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(@Model.Variable.NSEStatus))@VariableFunctions.NSERating(@Model.Variable.CurrentNSE)</td>
+                        <td class="stats-cell">&nbsp;&nbsp;</td>
+                    </tr>
+                    <tr>
+                        <td class="stats-cell">"RSR"</td>
+                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.CurrentRSR, 3, 0, isAccepted: false))</td>
+                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@Model.Variable.AcceptedRSR, 3, 0, isAccepted: true))</td>
+                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(@Model.Variable.RSRStatus))@VariableFunctions.RSRRating(@Model.Variable.CurrentNSE)</td>
+                    </tr>
+                </table>
+            </div>
+        </div>
+    </div>
+    <script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script>
+
+    <script type="text/javascript">
+
+        google.charts.load('current', { 'packages': ['corechart'] });
+        google.charts.setOnLoadCallback(drawChart);
+        function drawChart() {
+            var jsonData = $.ajax({
+                url: location.pathname + "?handler=ChartData&id=@Model.Id",
+                dataType: "json",
+                async: false
+            }).responseText;
+            
+            var data = new google.visualization.DataTable(jsonData);
+            var options = {
+                'title': '@Model.Name',
+                'width': 600,
+                'height': 600,
+                hAxis: { title: 'Observed' },
+                vAxis: { title: 'Predicted' },
+                legend: { position: 'right' },
+                trendlines: { 0: {}, 1: {} },    // Draw a trendline for data series 0 and 1
+                series: {
+                    2: { lineWidth: 1, pointSize: 0, color:'black' } // Series 2 (1:1) is a line.
+                }
+            };
+
+            var chart = new google.visualization.ScatterChart(document.getElementById('chart'));
+            chart.draw(data, options);
+        }
+    </script>
 }
\ No newline at end of file
diff --git a/APSIM.POStats.Portal/Pages/Chart.cshtml.cs b/APSIM.POStats.Portal/Pages/Chart.cshtml.cs
index cdcdff7..d999df1 100644
--- a/APSIM.POStats.Portal/Pages/Chart.cshtml.cs
+++ b/APSIM.POStats.Portal/Pages/Chart.cshtml.cs
@@ -54,8 +54,8 @@ public ActionResult OnGetChartData(int id)
 
             DataTable gdt = new DataTable();
             gdt.AddColumn(new Column(ColumnType.Number, "Observed", "Observed"));       // X
-            gdt.AddColumn(new Column(ColumnType.Number, "Accepted", "Accepted"));       // Accepted
-            gdt.AddColumn(new Column(ColumnType.Number, "Predicted", "Current"));       // Y
+            gdt.AddColumn(new Column(ColumnType.Number, "Accepted", "Old"));       // Y - old
+            gdt.AddColumn(new Column(ColumnType.Number, "Predicted", "New"));       // Y - new
 
             // Add in accepted values.
             VariableFunctions.GetData(accepted, out double[] acceptedPredicted, out double[] acceptedObserved, out string[] acceptedLabels);
@@ -69,8 +69,8 @@ public ActionResult OnGetChartData(int id)
                     {
                         var r = gdt.NewRow();
                         r.AddCell(new Cell(acceptedObserved[i], acceptedObserved[i].ToString("f3")));    // observed
-                        r.AddCell(new Cell(acceptedPredicted[i], $"{acceptedPredicted[i]:f3} ({acceptedLabels[i]})"));  // accepted
-                        r.AddCell(new Cell(null, null));    // current
+                        r.AddCell(new Cell(acceptedPredicted[i], $"{acceptedPredicted[i]:f3} ({acceptedLabels[i]})"));  // old
+                        r.AddCell(new Cell(null, null));    // new
                         gdt.AddRow(r);
                     }
                 }
@@ -84,8 +84,8 @@ public ActionResult OnGetChartData(int id)
             {
                 var r = gdt.NewRow();
                 r.AddCell(new Cell(observed[i], observed[i].ToString("f3")));           // observed
-                r.AddCell(new Cell(null, null));    // accepted
-                r.AddCell(new Cell(predicted[i], $"{predicted[i]:f3} ({labels[i]})"));  // current
+                r.AddCell(new Cell(null, null));    // old
+                r.AddCell(new Cell(predicted[i], $"{predicted[i]:f3} ({labels[i]})"));  // new
                 gdt.AddRow(r);
             }
 
diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml b/APSIM.POStats.Portal/Pages/Index.cshtml
index f5efbe7..ea2cbde 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml
@@ -25,11 +25,12 @@
             </tr>
             <tr class="ratings-table-heading">
                 <td class="ratings-table-column">&nbsp;</td>
-                <td class="ratings-table-column">Performance Rating</td>
-                <td class="ratings-table-column">RSR</td>
-                <td class="ratings-table-column">NSE</td>
-            </tr>
-            <tr class="ratings-table-row">
+                <td class="ratings-tab
+    using Microsoft.EntityFrameworkCore.Metadata.Internal;
+
+#nullable disable
+#line default
+        <tr class="ratings-table-row">
                 <td class="ratings-table-column">***</td>
                 <td class="ratings-table-column">Very Good</td>
                 <td class="ratings-table-column">0.00 ≤ RSR ≤ 0.50</td>
@@ -82,95 +83,109 @@
             <br />
             <br />
             <div>
+                <input type="hidden" name="stats-value" value="@Model.OnlyShowChangedStats.ToString()" readonly />
                 @if (Model.OnlyShowChangedStats)
                 {
-                    <input type="hidden" name="StatsLabel" value="Showing stats that have changed." readonly />
-                    <button name="toggle-button" type="submit">Show all stats</button>
+                    <button name="stats-button" type="submit" value="True">Show all stats</button>
                 }
                 else
                 {
-                    <input type="hidden" name="StatsLabel" value="Showing all stats." readonly />
-                    <button name="toggle-button" type="submit">Show changed stats</button>
+                    <button name="stats-button" type="submit" value="True">Show changed stats</button>
                 }
             </div>
+            <br />
+            <br />
             <div>
+                <input type="hidden" name="logs-value" value="@Model.ShowLogs.ToString()" readonly />
                 @if (Model.ShowLogs)
                 {
-                    <input type="hidden" name="LogsLabel" value="Hide Output Logs" readonly />
-                    <button name="toggle-button-logs" type="submit">Hide Output Logs</button>
+                    <button name="logs-button" type="submit" value="True">Hide Logs</button>
                 }
                 else
                 {
-                    <input type="hidden" name="LogsLabel" value="Show Output Logs" readonly />
-                    <button name="toggle-button-logs" type="submit">Show Output Logs</button>
+                    <button name="logs-button" type="submit" value="True">Show Logs</button>
                 }
             </div>
             <br />
             <br />
+            <div>
+                <label for="Filter">Filter By:</label>
+                <input name="filter-value" value="@Model.Filter" />
+                <button name="filter-button" type="submit" value="True">Enter</button>
+            </div>
         </form>
 
         <table class="stats-table">
             @{
-                var files = PullRequestFunctions.GetFileComparisons(Model.PullRequest);
+                List<ApsimFileComparison> files = PullRequestFunctions.GetFileComparisons(Model.PullRequest);
                 if (Model.OnlyShowChangedStats)
                     PullRequestFunctions.RemoveSame(files);
             }
             @foreach (var file in files)
             {
-                <tr></tr>
-                <tr>
-                    <td class="stats-table-heading" colspan="18">
-                        <b>@Html.Raw(IndexModel.EmitFileName(file))</b>
-                    </td>
-                </tr>
-                @foreach (var table in file.Tables)
+                @if (IndexModel.FilterVariables(files, Model.Filter, file.Name, "", ""))
                 {
+                    <tr></tr>
                     <tr>
-                        <td class=".stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
-                        <td class="stats-table-heading" colspan="2"><i>@Html.Raw(IndexModel.EmitTableName(@table))</i></td>
-                        <td class="stats-heading">&nbsp;&nbsp;</td>
-                        <td class="stats-heading" colspan="3">n</td>
-                        <td class="stats-heading">&nbsp;&nbsp;</td>
-                        <td class="stats-heading" colspan="3">RMSE</td>
-                        <td class="stats-heading">&nbsp;&nbsp;</td>
-                        <td class="stats-heading" colspan="3">NSE</td>
-                        <td class="stats-heading">&nbsp;&nbsp;</td>
-                        <td class="stats-heading" colspan="3">RSR</td>
+                        <td class="stats-table-heading" colspan="18">
+                            <b>@Html.Raw(IndexModel.EmitFileName(file))</b>
+                        </td>
                     </tr>
-
-                    @foreach (var variable in table.VariableComparisons)
+                    @foreach (var table in file.Tables)
                     {
-                        <tr>
-                            <td class="stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
-                            <td class="stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
-                            <td class="stats-variablename"><a asp-page="/Chart" asp-route-id="@variable.Id">@Html.Raw(IndexModel.EmitVariableName(@variable))</a></td>
-                            <td class="stats-cell">&nbsp;&nbsp;</td>
+                        @if (IndexModel.FilterVariables(files, Model.Filter, file.Name, table.Name, ""))
+                        {
+                            <tr>
+                                <td class=".stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
+                                <td class="stats-table-heading" colspan="2"><i>@Html.Raw(IndexModel.EmitTableName(@table))</i></td>
+                                <td class="stats-heading">&nbsp;&nbsp;</td>
+                                <td class="stats-heading" colspan="3">n</td>
+                                <td class="stats-heading">&nbsp;&nbsp;</td>
+                                <td class="stats-heading" colspan="3">RMSE</td>
+                                <td class="stats-heading">&nbsp;&nbsp;</td>
+                                <td class="stats-heading" colspan="3">NSE</td>
+                                <td class="stats-heading">&nbsp;&nbsp;</td>
+                                <td class="stats-heading" colspan="3">RSR</td>
+                            </tr>
+
+                            @foreach (var variable in table.VariableComparisons)
+                            {
+                                @if (IndexModel.FilterVariables(files, Model.Filter, file.Name, table.Name, variable.Name))
+                                {
+                                    <tr>
+                                        <td class="stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
+                                        <td class="stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
+                                        <td class="stats-variablename"><a asp-page="/Chart" asp-route-id="@variable.Id">@Html.Raw(IndexModel.EmitVariableName(@variable))</a></td>
+                                        <td class="stats-cell">&nbsp;&nbsp;</td>
 
-                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentN, isAccepted: false))</td>
-                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedN, isAccepted: true))</td>
-                            <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.NStatus))</td>
-                            <td class="stats-cell">&nbsp;&nbsp;</td>
+                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentN, isAccepted: false))</td>
+                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedN, isAccepted: true))</td>
+                                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.NStatus))</td>
+                                        <td class="stats-cell">&nbsp;&nbsp;</td>
 
-                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(variable.CurrentRMSE, 0, 6, isAccepted: false))</td>
-                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(variable.AcceptedRMSE, 0, 6, isAccepted: true))</td>
-                            <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.RMSEStatus))</td>
-                            <td class="stats-cell">&nbsp;&nbsp;</td>
+                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(variable.CurrentRMSE, 0, 6, isAccepted: false))</td>
+                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(variable.AcceptedRMSE, 0, 6, isAccepted: true))</td>
+                                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.RMSEStatus))</td>
+                                        <td class="stats-cell">&nbsp;&nbsp;</td>
 
-                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentNSE, 3, 0, isAccepted: false))</td>
-                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedNSE, 3, 0, isAccepted: true))</td>
-                            <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.NSEStatus))@VariableFunctions.NSERating(variable.CurrentNSE)</td>
-                            <td class="stats-cell">&nbsp;&nbsp;</td>
+                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentNSE, 3, 0, isAccepted: false))</td>
+                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedNSE, 3, 0, isAccepted: true))</td>
+                                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.NSEStatus))@VariableFunctions.NSERating(variable.CurrentNSE)</td>
+                                        <td class="stats-cell">&nbsp;&nbsp;</td>
 
-                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentRSR, 3, 0, isAccepted: false))</td>
-                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedRSR, 3, 0, isAccepted: true))</td>
-                            <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.RSRStatus))@VariableFunctions.RSRRating(variable.CurrentRSR)</td>
-                        </tr>
+                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentRSR, 3, 0, isAccepted: false))</td>
+                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedRSR, 3, 0, isAccepted: true))</td>
+                                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.RSRStatus))@VariableFunctions.RSRRating(variable.CurrentRSR)</td>
+                                    </tr>
+                                }
+                            }
+                        }
                     }
                 }
             }
         </table>
         <div class="output-logs">
-            logs:
+            Logs:
             @if (Model.ShowLogs) 
             {
                 @foreach (string line in @Model.PullRequest.Output.Split("\n"))
diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml.cs b/APSIM.POStats.Portal/Pages/Index.cshtml.cs
index 670de26..34adfbd 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml.cs
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml.cs
@@ -4,6 +4,7 @@
 using APSIM.Shared.Utilities;
 using Microsoft.AspNetCore.Mvc.RazorPages;
 using System;
+using System.Collections.Generic;
 using System.Linq;
 
 namespace APSIM.POStats.Portal.Pages
@@ -27,7 +28,7 @@ public IndexModel(StatsDbContext stats)
         public bool OnlyShowChangedStats { get; set; } = true;
 
         /// <summary>Show standard output logs</summary>
-        public bool ShowLogs { get; set; } = true;
+        public bool ShowLogs { get; set; } = false;
 
         /// <summary>The pull request being analysed.</summary>
         public PullRequestDetails PullRequest { get; private set; }
@@ -35,6 +36,8 @@ public IndexModel(StatsDbContext stats)
         /// <summary>The Url for the web site.</summary>
         public string BaseUrl { get { return $"{HttpContext.Request.Scheme}://{HttpContext.Request.Host}"; } }
 
+        public string Filter { get; set; } = "";
+
         /// <summary>Invoked when page is first loaded.</summary>
         /// <param name="pullRequestNumber">The pull request to work with.</param>
         public void OnGet(int pullRequestNumber)
@@ -53,11 +56,15 @@ public void OnPost()
             if (PullRequest == null)
                 throw new Exception($"Cannot find pull request #{pullRequestNumber} in stats database");
 
-            var statsLabel = Request.Form["StatsLabel"].ToString();
-            OnlyShowChangedStats = statsLabel == "Showing all stats.";
+            this.OnlyShowChangedStats = Request.Form["stats-value"].ToString().ToLower() == "true";
+            this.ShowLogs = Request.Form["logs-value"].ToString().ToLower() == "true";
+            this.Filter = Request.Form["filter-value"].ToString();
+
+            if (!String.IsNullOrEmpty(Request.Form["stats-button"].ToString()))
+                this.OnlyShowChangedStats = !this.OnlyShowChangedStats;
 
-            var logsLabel = Request.Form["LogsLabel"].ToString();
-            ShowLogs = logsLabel == "Show Output Logs";
+            if (!String.IsNullOrEmpty(Request.Form["logs-button"].ToString()))
+                this.ShowLogs = !this.ShowLogs;
         }
 
         /// <summary>Emit html to display tick/cross.</summary>
@@ -155,5 +162,35 @@ public static string EmitFileName(ApsimFileComparison file)
             else
                 return file.Name;
         }
+
+        public static bool FilterVariables(List<ApsimFileComparison> files, string filter, string fileName, string tableName = "", string variableName = "")
+        {
+            if (String.IsNullOrEmpty(filter))
+                return true;
+                
+            foreach (ApsimFileComparison file in files)
+            {
+                if (file.Name == fileName)
+                {
+                    foreach (TableComparison table in file.Tables)
+                    {
+                        if (String.IsNullOrEmpty(tableName) || tableName == table.Name)
+                        {
+                            foreach (VariableComparison variable in table.VariableComparisons)
+                            {
+                                if (String.IsNullOrEmpty(variableName) || variableName == variable.Name)
+                                {
+                                    if (variable.Name.ToLower().Contains(filter.ToLower()))
+                                    {
+                                        return true;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            return false;
+        }
     }
 }
\ No newline at end of file
diff --git a/APSIM.POStats.Portal/Pages/Shared/_Layout.cshtml b/APSIM.POStats.Portal/Pages/Shared/_Layout.cshtml
index bdf43e7..c2b1b0c 100644
--- a/APSIM.POStats.Portal/Pages/Shared/_Layout.cshtml
+++ b/APSIM.POStats.Portal/Pages/Shared/_Layout.cshtml
@@ -1,50 +1,50 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>@ViewData["Title"] - APSIM.POStats.Portal</title>
-    <link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
-    <link rel="stylesheet" href="~/css/site.css" />
-</head>
-<body>
-    <header>
-        <nav class="navbar navbar-expand-sm navbar-toggleable-sm navbar-light bg-white border-bottom box-shadow mb-3">
-            <div class="container">
-                <a class="navbar-brand" asp-area="" asp-page="/Index">APSIM.POStats.Portal</a>
-                <button class="navbar-toggler" type="button" data-toggle="collapse" data-target=".navbar-collapse" aria-controls="navbarSupportedContent"
-                        aria-expanded="false" aria-label="Toggle navigation">
-                    <span class="navbar-toggler-icon"></span>
-                </button>
-                <div class="navbar-collapse collapse d-sm-inline-flex flex-sm-row-reverse">
-                    <ul class="navbar-nav flex-grow-1">
-                        <li class="nav-item">
-                            <a class="nav-link text-dark" asp-area="" asp-page="/Index">Table</a>
-                        </li>
-                        <li class="nav-item">
-                            <a class="nav-link text-dark" asp-area="" asp-page="/Heatmap" asp-route-id=@ViewData["ID"]>Heatmap</a>
-                        </li>
-                        <li class="nav-item">
-                            <a class="nav-link text-dark" asp-area="" asp-page="/Accept" asp-route-id=@ViewData["ID"]>Accept stats</a>
-                        </li>
-                        <li class="nav-item">
-                            <a class="nav-link text-dark" asp-area="" asp-page="/UpdateAccepted" asp-route-id=@ViewData["ID"]>Update stats</a>
-                        </li>
-                    </ul>
-                </div>
-            </div>
-        </nav>
-    </header>
-    <div class="container">
-        <main role="main" class="pb-3">
-            @RenderBody()
-        </main>
-    </div>
-
-    <script src="~/lib/jquery/dist/jquery.min.js"></script>
-    <script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js"></script>
-    <script src="~/js/site.js" asp-append-version="true"></script>
-
-    @RenderSection("Scripts", required: false)
-</body>
-</html>
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>@ViewData["Title"] - APSIM.POStats.Portal</title>
+    <link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
+    <link rel="stylesheet" href="~/css/site.css" />
+</head>
+<body>
+    <header>
+        <nav class="navbar navbar-expand-sm navbar-toggleable-sm navbar-light bg-white border-bottom box-shadow mb-3">
+            <div class="container-fluid">
+                <a class="navbar-brand" asp-area="" asp-page="/Index">APSIM.POStats.Portal</a>
+                <button class="navbar-toggler" type="button" data-toggle="collapse" data-target=".navbar-collapse" aria-controls="navbarSupportedContent"
+                        aria-expanded="false" aria-label="Toggle navigation">
+                    <span class="navbar-toggler-icon"></span>
+                </button>
+                <div class="navbar-collapse collapse d-sm-inline-flex flex-sm-row-reverse">
+                    <ul class="navbar-nav flex-grow-1">
+                        <li class="nav-item">
+                            <a class="nav-link text-dark" asp-area="" asp-page="/Index">Table</a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link text-dark" asp-area="" asp-page="/Heatmap" asp-route-id=@ViewData["ID"]>Heatmap</a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link text-dark" asp-area="" asp-page="/Accept" asp-route-id=@ViewData["ID"]>Accept stats</a>
+                        </li>
+                        <li class="nav-item">
+                            <a class="nav-link text-dark" asp-area="" asp-page="/UpdateAccepted" asp-route-id=@ViewData["ID"]>Update stats</a>
+                        </li>
+                    </ul>
+                </div>
+            </div>
+        </nav>
+    </header>
+    <div class="container-fluid">
+        <main role="main" class="pb-3">
+            @RenderBody()
+        </main>
+    </div>
+
+    <script src="~/lib/jquery/dist/jquery.min.js"></script>
+    <script src="~/lib/bootstrap/dist/js/bootstrap.bundle.min.js"></script>
+    <script src="~/js/site.js" asp-append-version="true"></script>
+
+    @RenderSection("Scripts", required: false)
+</body>
+</html>

From b74fd1ab477aa3c8b317abed120dfa50307a1d50 Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Thu, 7 Aug 2025 13:33:08 +1000
Subject: [PATCH 07/48] attempt to work aorund transient errors

---
 .../APSIM.POStats.Collector.csproj            |  4 +-
 .../APSIM.POStats.Portal.csproj               | 16 ++---
 APSIM.POStats.Portal/Controllers/Api.cs       |  4 +-
 APSIM.POStats.Portal/Startup.cs               | 19 ++++-
 .../APSIM.POStats.Shared.csproj               |  8 +--
 APSIM.POStats.Shared/StatsDbContext.cs        | 69 +++++++++++++++----
 .../APSIM.POStats.Tests.csproj                | 14 ++--
 build.sh                                      |  2 +-
 upload-dummy-pullrequest.sh                   | 12 ++--
 9 files changed, 105 insertions(+), 43 deletions(-)

diff --git a/APSIM.POStats.Collector/APSIM.POStats.Collector.csproj b/APSIM.POStats.Collector/APSIM.POStats.Collector.csproj
index 0c949ef..5627d2f 100644
--- a/APSIM.POStats.Collector/APSIM.POStats.Collector.csproj
+++ b/APSIM.POStats.Collector/APSIM.POStats.Collector.csproj
@@ -6,8 +6,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="2.1.9" />
-    <PackageReference Include="System.Configuration.ConfigurationManager" Version="8.0.0" />
+    <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.0" />
+    <PackageReference Include="System.Configuration.ConfigurationManager" Version="8.0.1" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/APSIM.POStats.Portal/APSIM.POStats.Portal.csproj b/APSIM.POStats.Portal/APSIM.POStats.Portal.csproj
index 833bfca..3d040b0 100644
--- a/APSIM.POStats.Portal/APSIM.POStats.Portal.csproj
+++ b/APSIM.POStats.Portal/APSIM.POStats.Portal.csproj
@@ -10,17 +10,17 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Google.DataTable.Net.Wrapper" Version="4.0.0" />
-    <PackageReference Include="Microsoft.Data.Sqlite.Core" Version="6.0.33" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.33" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Proxies" Version="6.0.33" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.33" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.33">
+    <PackageReference Include="Google.DataTable.Net.Wrapper" Version="4.1.0" />
+    <PackageReference Include="Microsoft.Data.Sqlite.Core" Version="8.0.18" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.18" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Proxies" Version="8.0.18" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.18" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.18">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="2.1.9" />
-    <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="6.0.3" />
+    <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.0" />
+    <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="8.0.3" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index c693bec..32ea732 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -116,7 +116,7 @@ public IActionResult Close(int pullrequestnumber, string commitid)
         /// <returns></returns>
         [HttpPost("adddata")]
         [RequestSizeLimit(100_000_000)]
-        public IActionResult Post([FromBody]PullRequestDetails pullrequest)
+        public async Task<IActionResult> PostAsync([FromBody]PullRequestDetails pullrequest)
         {
             Console.WriteLine($"api/adddata called");
 
@@ -128,7 +128,7 @@ public IActionResult Post([FromBody]PullRequestDetails pullrequest)
                 Console.WriteLine($"Adding Data to PR \"{pullrequest.PullRequest}\"");
                 try
                 {
-                    PullRequestDetails pr = statsDb.AddDataToPullRequest(pullrequest);
+                    PullRequestDetails pr = await statsDb.AddDataToPullRequest(pullrequest);
                     Console.WriteLine($"{pr.CountReturned} of {pr.CountTotal} completed.");
                     if (pr.CountReturned == pr.CountTotal)
                     {
diff --git a/APSIM.POStats.Portal/Startup.cs b/APSIM.POStats.Portal/Startup.cs
index 4823da5..7ca82da 100644
--- a/APSIM.POStats.Portal/Startup.cs
+++ b/APSIM.POStats.Portal/Startup.cs
@@ -3,9 +3,11 @@
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Sqlite.Infrastructure.Internal;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 
+
 namespace APSIM.POStats.Portal
 {
     public class Startup
@@ -34,11 +36,26 @@ public void ConfigureServices(IServiceCollection services)
                 throw new Exception("Cannot find environment variable PORTAL_DB");
 
             if (connectionString.Contains(".db"))
+            {
+                Console.WriteLine("Using SQLite database");
                 services.AddDbContext<StatsDbContext>(options => options.UseLazyLoadingProxies().UseSqlite(connectionString));
+            }
             else
             {
+                Console.WriteLine("Using MySQL database");
                 var serverVersion = new MySqlServerVersion(new Version(10, 0, 0));
-                services.AddDbContext<StatsDbContext>(options => options.UseLazyLoadingProxies().UseMySql(connectionString, serverVersion));
+                services.AddDbContext<StatsDbContext>(options => options.UseLazyLoadingProxies().UseMySql(
+                    connectionString,
+                    serverVersion,
+                    mySqlOptionsAction: sqlOptions =>
+                    {
+                        sqlOptions.CommandTimeout(60);
+                        sqlOptions.EnableRetryOnFailure(
+                            maxRetryCount: 5,
+                            maxRetryDelay: TimeSpan.FromSeconds(30),
+                            errorNumbersToAdd: null);
+                    })
+                );
             }
         }
 
diff --git a/APSIM.POStats.Shared/APSIM.POStats.Shared.csproj b/APSIM.POStats.Shared/APSIM.POStats.Shared.csproj
index 540b7b2..8b9cec2 100644
--- a/APSIM.POStats.Shared/APSIM.POStats.Shared.csproj
+++ b/APSIM.POStats.Shared/APSIM.POStats.Shared.csproj
@@ -9,11 +9,11 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Data.Sqlite.Core" Version="6.0.33" />
+    <PackageReference Include="Microsoft.Data.Sqlite.Core" Version="8.0.18" />
     <PackageReference Include="System.Data.DataSetExtensions" Version="4.5.0" />
-    <PackageReference Include="System.Text.Json" Version="8.0.5" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="6.0.33" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.33" />
+    <PackageReference Include="System.Text.Json" Version="8.0.6" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.18" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.18" />
   </ItemGroup>
 
 </Project>
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 212ba93..4ed2582 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -2,6 +2,8 @@
 using Microsoft.EntityFrameworkCore;
 using System;
 using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace APSIM.POStats.Shared
 {
@@ -64,25 +66,68 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
         /// </remarks>
         /// <param name="pullRequest">The pull request to copy the data from..</param>
         /// <returns>Reference to stored PullRequest</returns>
-        public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullRequest)
+        public async Task<PullRequestDetails> AddDataToPullRequest(PullRequestDetails fromPullRequest, int retryCount = 0)
         {
-            // Find the pull request. Should always exist if OpenPullRequest has been called.
-            var pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == fromPullRequest.PullRequest)
-                     ?? throw new Exception($"Cannot find POStats pull request number: {fromPullRequest.PullRequest}");
 
-            pr.CountReturned += 1;
+            var pr = new PullRequestDetails();
+            try
+            {
+                // Find the pull request. Should always exist if OpenPullRequest has been called.
+                pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == fromPullRequest.PullRequest)
+                        ?? throw new Exception($"Cannot find POStats pull request number: {fromPullRequest.PullRequest}");
 
-            foreach (ApsimFile file in fromPullRequest.Files)
-                Console.WriteLine($"File \"{file.Name}\" added to PR {fromPullRequest.PullRequest}");
+                pr.CountReturned += 1;
 
-            if (fromPullRequest.Files != null)
-                pr.Files.AddRange(fromPullRequest.Files);
+                foreach (ApsimFile file in fromPullRequest.Files)
+                    Console.WriteLine($"File \"{file.Name}\" added to PR {fromPullRequest.PullRequest}");
 
-            pr.Output += fromPullRequest.Output;
+                if (fromPullRequest.Files != null)
+                    pr.Files.AddRange(fromPullRequest.Files);
 
-            SaveChanges();
+                pr.Output += fromPullRequest.Output;
 
-            return pr;
+                await SaveChangesAsync();
+
+                return pr;
+            }
+            catch (Exception ex)
+            {
+                // Console.WriteLine($"Retry Number: {retryCount} An issue was encountered while adding data to the pull request: " + ex.ToString());
+                // if (retryCount < 5)
+                // {
+                //     // Wait for a random amount of time before retrying
+                //     Console.WriteLine("Retrying to add data to pull request...");
+                //     Thread.Sleep(new Random().Next(100, 1000));
+                //     AddDataToPullRequest(fromPullRequest, retryCount + 1);
+                // }
+                // Console.WriteLine("Add data retry attempts exceeded: " + ex.ToString());
+                // throw;
+                try
+                {
+                    var wait = new Random().Next(5000, 10000);
+                    Thread.Sleep(wait);
+                    Console.WriteLine("Unable to add data to pull request, retrying in " + wait + "ms");
+                    await SaveChangesAsync();
+                    return pr;
+                }
+                catch (Exception)
+                {
+                    try
+                    {
+                        var wait = new Random().Next(100, 1000);
+                        Thread.Sleep(wait);
+                        Console.WriteLine("Unable to add data again to pull request, retrying in " + wait + "ms");
+                        await SaveChangesAsync();
+                        return pr;
+                    }
+                    catch (Exception)
+                    {
+                        Console.WriteLine("Unable to add data to pull request after retries: " + ex.ToString());
+                        throw new Exception("Unable to add data to pull request after two retries", ex);
+                    }
+                }
+                throw new Exception("Unable to add data: " + ex.ToString());
+            }
         }
 
         /// <summary>
diff --git a/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj b/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj
index 7c395b9..efc24b4 100644
--- a/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj
+++ b/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj
@@ -10,13 +10,13 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="6.0.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
-    <PackageReference Include="NUnit" Version="3.14.0" />
-    <PackageReference Include="NUnit.Analyzers" Version="3.9.0" />
-    <PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
-    <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="2.1.9" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="6.0.33" />
+    <PackageReference Include="coverlet.collector" Version="6.0.4" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
+    <PackageReference Include="NUnit" Version="4.3.2" />
+    <PackageReference Include="NUnit.Analyzers" Version="4.9.2" />
+    <PackageReference Include="NUnit3TestAdapter" Version="5.0.0" />
+    <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.0" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.18" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/build.sh b/build.sh
index 41df470..798e55e 100644
--- a/build.sh
+++ b/build.sh
@@ -4,6 +4,6 @@
 
 docker build --target postats-build -t postats-build .
 docker build --target postats-collector -t apsiminitiative/postats-collector .
-docker build --target postats-portal -t apsiminitiative/postats-portal .
+docker build --target postats-portal -t apsiminitiative/postats2-portal .
 
 docker rmi postats-build
\ No newline at end of file
diff --git a/upload-dummy-pullrequest.sh b/upload-dummy-pullrequest.sh
index 9b98749..6a56b05 100644
--- a/upload-dummy-pullrequest.sh
+++ b/upload-dummy-pullrequest.sh
@@ -2,17 +2,17 @@
 
 echo Opening pull request
 curl --verbose \
-     'http://localhost:8001/api/open?pullRequestNumber=1234&author=author_name'
+     'http://localhost:8081/api/open?pullRequestNumber=1234&author=author_name&commitid=123456&count=100'
 echo
 
 echo Adding data.
 curl --verbose \
      --header "Content-Type: application/json" \
      --data-binary "@dummy-pullrequest.json" \
-     http://localhost:8001/api/adddata
+     http://localhost:8081/api/adddata
 echo
 
-echo Closing pull request
-curl --verbose \
-      'http://localhost:8001/api/close?pullRequestNumber=1234'
-echo
\ No newline at end of file
+# echo Closing pull request
+# curl --verbose \
+#       'http://localhost:8081/api/close?pullRequestNumber=1234'
+# echo
\ No newline at end of file

From 2562e27e8227538bca83f6d2a5cd80ad4ca1bab8 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Thu, 7 Aug 2025 13:46:23 +1000
Subject: [PATCH 08/48] Refactor to recursive function

---
 APSIM.POStats.Shared/StatsDbContext.cs | 70 ++++++++++----------------
 1 file changed, 26 insertions(+), 44 deletions(-)

diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 4ed2582..d8051ee 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -68,65 +68,47 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
         /// <returns>Reference to stored PullRequest</returns>
         public async Task<PullRequestDetails> AddDataToPullRequest(PullRequestDetails fromPullRequest, int retryCount = 0)
         {
-
             var pr = new PullRequestDetails();
-            try
-            {
-                // Find the pull request. Should always exist if OpenPullRequest has been called.
-                pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == fromPullRequest.PullRequest)
-                        ?? throw new Exception($"Cannot find POStats pull request number: {fromPullRequest.PullRequest}");
 
-                pr.CountReturned += 1;
+            // Find the pull request. Should always exist if OpenPullRequest has been called.
+            pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == fromPullRequest.PullRequest)
+                    ?? throw new Exception($"Cannot find POStats pull request number: {fromPullRequest.PullRequest}");
 
-                foreach (ApsimFile file in fromPullRequest.Files)
-                    Console.WriteLine($"File \"{file.Name}\" added to PR {fromPullRequest.PullRequest}");
+            pr.CountReturned += 1;
 
-                if (fromPullRequest.Files != null)
-                    pr.Files.AddRange(fromPullRequest.Files);
+            foreach (ApsimFile file in fromPullRequest.Files)
+                Console.WriteLine($"File \"{file.Name}\" added to PR {fromPullRequest.PullRequest}");
 
-                pr.Output += fromPullRequest.Output;
+            if (fromPullRequest.Files != null)
+                pr.Files.AddRange(fromPullRequest.Files);
 
-                await SaveChangesAsync();
+            pr.Output += fromPullRequest.Output;
+
+            SaveChangesMultipleTries();
+
+            return pr;
+        }
 
-                return pr;
+        public bool SaveChangesMultipleTries(int retries = 0)
+        {
+            try
+            {
+                SaveChanges();
+                return true;
             }
             catch (Exception ex)
             {
-                // Console.WriteLine($"Retry Number: {retryCount} An issue was encountered while adding data to the pull request: " + ex.ToString());
-                // if (retryCount < 5)
-                // {
-                //     // Wait for a random amount of time before retrying
-                //     Console.WriteLine("Retrying to add data to pull request...");
-                //     Thread.Sleep(new Random().Next(100, 1000));
-                //     AddDataToPullRequest(fromPullRequest, retryCount + 1);
-                // }
-                // Console.WriteLine("Add data retry attempts exceeded: " + ex.ToString());
-                // throw;
-                try
+                if (retries < 10)
                 {
-                    var wait = new Random().Next(5000, 10000);
-                    Thread.Sleep(wait);
+                    var wait = new Random().Next(1000, 5000);
                     Console.WriteLine("Unable to add data to pull request, retrying in " + wait + "ms");
-                    await SaveChangesAsync();
-                    return pr;
+                    Thread.Sleep(wait);
+                    return SaveChangesMultipleTries(retries + 1);
                 }
-                catch (Exception)
+                else
                 {
-                    try
-                    {
-                        var wait = new Random().Next(100, 1000);
-                        Thread.Sleep(wait);
-                        Console.WriteLine("Unable to add data again to pull request, retrying in " + wait + "ms");
-                        await SaveChangesAsync();
-                        return pr;
-                    }
-                    catch (Exception)
-                    {
-                        Console.WriteLine("Unable to add data to pull request after retries: " + ex.ToString());
-                        throw new Exception("Unable to add data to pull request after two retries", ex);
-                    }
+                    throw new Exception(ex.Message);
                 }
-                throw new Exception("Unable to add data: " + ex.ToString());
             }
         }
 

From b9787a69d5c78484c4c5534c9b0105e99600f993 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Thu, 7 Aug 2025 14:39:30 +1000
Subject: [PATCH 09/48] Remove github status updates for each adddata call

---
 APSIM.POStats.Portal/Controllers/Api.cs | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index 32ea732..0c98357 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -50,7 +50,7 @@ public IActionResult Open(int pullrequestnumber, string commitid, int count, str
                 return BadRequest("You must supply an author");
             try
             {
-                GitHub.SetStatus(pullrequestnumber, commitid, VariableComparison.Status.Running, $"Running: 0 of {count} completed");
+                GitHub.SetStatus(pullrequestnumber, commitid, VariableComparison.Status.Running, $"Running {count} Validation Tasks");
 
                 statsDb.OpenPullRequest(pullrequestnumber, commitid, author, count);
 
@@ -136,11 +136,6 @@ public async Task<IActionResult> PostAsync([FromBody]PullRequestDetails pullrequ
                         Task<string> response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={pr.PullRequest}&commitId={pr.Commit}");
                         response.Wait();
                     }
-                    else
-                    {
-                        string message = $"Running: {pr.CountReturned} of {pr.CountTotal} completed";
-                        GitHub.SetStatus(pr.PullRequest, pr.Commit, VariableComparison.Status.Running, message);
-                    }
                 }
                 catch (Exception ex)
                 {

From 13de03ee6082e8455b85a2ac0376a3991284b5b6 Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Fri, 15 Aug 2025 16:05:01 +1000
Subject: [PATCH 10/48] changes to handle write retries

---
 APSIM.POStats.Collector/Program.cs      | 14 +++++-----
 APSIM.POStats.Portal/Controllers/Api.cs |  4 +--
 APSIM.POStats.Portal/Startup.cs         |  7 ++---
 APSIM.POStats.Shared/Collector.cs       | 36 +++++++++++--------------
 APSIM.POStats.Shared/StatsDbContext.cs  |  7 ++---
 build.sh                                |  2 +-
 6 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/APSIM.POStats.Collector/Program.cs b/APSIM.POStats.Collector/Program.cs
index 495216b..2c7ad0c 100644
--- a/APSIM.POStats.Collector/Program.cs
+++ b/APSIM.POStats.Collector/Program.cs
@@ -52,18 +52,20 @@ static int Main(string[] args)
                 //get the run date
                 DateTime runDate = DateTime.ParseExact(args[4], "yyyy.M.d-HH:mm", CultureInfo.CurrentCulture, DateTimeStyles.AssumeLocal);
 
-                //get the directories
-                List<string> searchDirectories = new List<string>();
+                //get the file paths
+                List<string> filePaths = new List<string>();
                 for (int i = 5; i < args.Length; i++)
                 {
-                    if (Directory.Exists(args[i]))
-                        searchDirectories.Add(args[i]);
+                    if (File.Exists(args[i]))
+                    {
+                        filePaths.Add(args[i]);
+                    }
                     else
-                        throw new Exception($"Directory \"{args[i]}\" does not exist.");
+                        throw new Exception($"File \"{args[i]}\" does not exist.");
                 }
 
                 //get the Pull Request details
-                PullRequestDetails pullRequest = Shared.Collector.RetrieveData(pullId, commitId, author, runDate, searchDirectories);
+                PullRequestDetails pullRequest = Shared.Collector.RetrieveData(pullId, commitId, author, runDate, filePaths);
 
                 string url = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
                 Console.WriteLine($"{url}");
diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index 0c98357..fb19b1c 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -116,7 +116,7 @@ public IActionResult Close(int pullrequestnumber, string commitid)
         /// <returns></returns>
         [HttpPost("adddata")]
         [RequestSizeLimit(100_000_000)]
-        public async Task<IActionResult> PostAsync([FromBody]PullRequestDetails pullrequest)
+        public IActionResult PostAsync([FromBody]PullRequestDetails pullrequest)
         {
             Console.WriteLine($"api/adddata called");
 
@@ -128,7 +128,7 @@ public async Task<IActionResult> PostAsync([FromBody]PullRequestDetails pullrequ
                 Console.WriteLine($"Adding Data to PR \"{pullrequest.PullRequest}\"");
                 try
                 {
-                    PullRequestDetails pr = await statsDb.AddDataToPullRequest(pullrequest);
+                    PullRequestDetails pr = statsDb.AddDataToPullRequest(pullrequest);
                     Console.WriteLine($"{pr.CountReturned} of {pr.CountTotal} completed.");
                     if (pr.CountReturned == pr.CountTotal)
                     {
diff --git a/APSIM.POStats.Portal/Startup.cs b/APSIM.POStats.Portal/Startup.cs
index 7ca82da..e11ce85 100644
--- a/APSIM.POStats.Portal/Startup.cs
+++ b/APSIM.POStats.Portal/Startup.cs
@@ -50,10 +50,11 @@ public void ConfigureServices(IServiceCollection services)
                     mySqlOptionsAction: sqlOptions =>
                     {
                         sqlOptions.CommandTimeout(60);
-                        sqlOptions.EnableRetryOnFailure(
-                            maxRetryCount: 5,
+                        sqlOptions.EnableRetryOnFailure(                           
+                            maxRetryCount: 20,
                             maxRetryDelay: TimeSpan.FromSeconds(30),
-                            errorNumbersToAdd: null);
+                            errorNumbersToAdd: null
+                        );
                     })
                 );
             }
diff --git a/APSIM.POStats.Shared/Collector.cs b/APSIM.POStats.Shared/Collector.cs
index e8f189f..f915af6 100644
--- a/APSIM.POStats.Shared/Collector.cs
+++ b/APSIM.POStats.Shared/Collector.cs
@@ -45,29 +45,25 @@ public static PullRequestDetails RetrieveData(int pullId, string commitId, strin
             pullRequest.Output = "";
             foreach (var filePath in filePaths)
             {
-                string currentPath = filePath.Trim();
-                DirectoryInfo info = new DirectoryInfo(@currentPath);
-                foreach (FileInfo fileInfo in info.GetFiles("*.apsimx", SearchOption.AllDirectories))
+                try
                 {
-                    try
+                    var stopwatch = Stopwatch.StartNew();
+                    var apsimFile = new ApsimFile()
                     {
-                        var stopwatch = Stopwatch.StartNew();
-                        var apsimFile = new ApsimFile()
-                        {
-                            Name = Path.GetFileNameWithoutExtension(fileInfo.FullName),
-                            PullRequest = pullRequest,
-                            PullRequestId = pullRequest.Id,
-                            Tables = GetTablesFromFile(fileInfo.FullName)
-                        };
-                        if (apsimFile.Tables.Count > 0)
-                            pullRequest.Files.Add(apsimFile);
-                        Console.WriteLine($"Read PO data from {fileInfo.FullName} in {stopwatch.Elapsed.Seconds} second(s).");
-                    }
-                    catch (Exception ex)
-                    {
-                        errorMessages += ex.ToString() + "\n";
-                    }
+                        Name = Path.GetFileNameWithoutExtension(filePath),
+                        PullRequest = pullRequest,
+                        PullRequestId = pullRequest.Id,
+                        Tables = GetTablesFromFile(filePath)
+                    };
+                    if (apsimFile.Tables.Count > 0)
+                        pullRequest.Files.Add(apsimFile);
+                    Console.WriteLine($"Read PO data from {filePath} in {stopwatch.Elapsed.Seconds} second(s).");
+                }
+                catch (Exception ex)
+                {
+                    errorMessages += ex.ToString() + "\n";
                 }
+                
             }
 
             FileInfo[] files = new DirectoryInfo("./").GetFiles("*stdout.txt", SearchOption.AllDirectories);
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index d8051ee..6995ef9 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -66,7 +66,7 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
         /// </remarks>
         /// <param name="pullRequest">The pull request to copy the data from..</param>
         /// <returns>Reference to stored PullRequest</returns>
-        public async Task<PullRequestDetails> AddDataToPullRequest(PullRequestDetails fromPullRequest, int retryCount = 0)
+        public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullRequest, int retryCount = 0)
         {
             var pr = new PullRequestDetails();
 
@@ -84,7 +84,8 @@ public async Task<PullRequestDetails> AddDataToPullRequest(PullRequestDetails fr
 
             pr.Output += fromPullRequest.Output;
 
-            SaveChangesMultipleTries();
+            // SaveChangesMultipleTries();
+            SaveChanges();
 
             return pr;
         }
@@ -98,7 +99,7 @@ public bool SaveChangesMultipleTries(int retries = 0)
             }
             catch (Exception ex)
             {
-                if (retries < 10)
+                if (retries < 5)
                 {
                     var wait = new Random().Next(1000, 5000);
                     Console.WriteLine("Unable to add data to pull request, retrying in " + wait + "ms");
diff --git a/build.sh b/build.sh
index 798e55e..15e0968 100644
--- a/build.sh
+++ b/build.sh
@@ -3,7 +3,7 @@
 #Run this script to build the docker images localled. Not used by github actions.
 
 docker build --target postats-build -t postats-build .
-docker build --target postats-collector -t apsiminitiative/postats-collector .
+docker build --target postats-collector -t apsiminitiative/postats2-collector .
 docker build --target postats-portal -t apsiminitiative/postats2-portal .
 
 docker rmi postats-build
\ No newline at end of file

From b8257f143b1e96e1c0f4a651ae1c6620f1ac67d2 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Thu, 21 Aug 2025 14:29:01 +1000
Subject: [PATCH 11/48] Making timer work better

---
 APSIM.POStats.Portal/Controllers/Api.cs | 84 +++++++++++++++----------
 APSIM.POStats.Shared/StatsDbContext.cs  |  5 --
 APSIM.POStats.Shared/TimeoutTimer.cs    |  7 ++-
 3 files changed, 57 insertions(+), 39 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index fb19b1c..0b4d098 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -15,16 +15,39 @@ public class Api : ControllerBase
         /// <summary>The database context.</summary>
         private readonly StatsDbContext statsDb;
 
-        /// <summary>Event handler for timeout timer.</summary>
-        private static void OnTimeout(Object source, ElapsedEventArgs e)
+        /// <summary>Lock for adding to the db</summary>
+        private readonly object _lock = new();
+
+        /// <summary>Event handler for finish timer.</summary>
+        private void OnCheckIfFinished(Object source, ElapsedEventArgs e)
         {
-            string url = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
-            if (string.IsNullOrEmpty(url))
-                throw new Exception($"Cannot find environment variable POSTATS_UPLOAD_URL");
+            Console.WriteLine($"Tick");
+            PullRequestTimer timer = source as PullRequestTimer;
+            timer.Stop();
+
+            lock (_lock)
+            {
+                PullRequestDetails pr = statsDb.GetPullRequest(timer.PullRequestNumber);
+                double minutes = (DateTime.Now - timer.StartTime).TotalMinutes;
+                Console.WriteLine($"{pr.PullRequest} ({pr.Commit}): {pr.Files.Count} of {pr.CountTotal} completed in {Math.Round(minutes, 1)} minutes");
+
+                if (pr.Commit == timer.CommitId)
+                {
+                    if (pr.Files.Count >= pr.CountTotal || minutes >= 40)
+                    {
+                        string url = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
+                        if (string.IsNullOrEmpty(url))
+                            throw new Exception($"Cannot find environment variable POSTATS_UPLOAD_URL");
 
-            TimeoutTimer timer = source as TimeoutTimer;
-            Task<string> response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequestNumber}&commitId={timer.CommitId}");
-            response.Wait();
+                        Task<string> response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequestNumber}&commitId={timer.CommitId}");
+                        response.Wait();
+                    }
+                    else
+                    {
+                        timer.Start();
+                    }
+                }
+            }
         }
 
         /// <summary>Constructor.</summary>
@@ -54,11 +77,12 @@ public IActionResult Open(int pullrequestnumber, string commitid, int count, str
 
                 statsDb.OpenPullRequest(pullrequestnumber, commitid, author, count);
 
-                // Create a timer to close the PR after 30 minutes
-                TimeoutTimer timeoutTimer = new TimeoutTimer {Interval=3600000, PullRequestNumber=pullrequestnumber, CommitId=commitid};
-                timeoutTimer.Elapsed += OnTimeout;
-                timeoutTimer.AutoReset = false;
-                timeoutTimer.Start();
+                // Create a timer to check how many files have been returned
+                PullRequestTimer finishTimer = new PullRequestTimer {Interval=10000, PullRequestNumber=pullrequestnumber, CommitId=commitid};
+                finishTimer.Elapsed += new ElapsedEventHandler(OnCheckIfFinished);
+                finishTimer.AutoReset = true;
+                finishTimer.StartTime = DateTime.Now;
+                finishTimer.Start();
             }
             catch (Exception ex)
             {
@@ -123,34 +147,28 @@ public IActionResult PostAsync([FromBody]PullRequestDetails pullrequest)
             if (pullrequest == null)
                 return BadRequest("You must supply a pull request");
 
-            if (statsDb.PullRequestWithCommitExists(pullrequest.PullRequest, pullrequest.Commit))
+            lock (_lock)
             {
-                Console.WriteLine($"Adding Data to PR \"{pullrequest.PullRequest}\"");
-                try
+                if (statsDb.PullRequestWithCommitExists(pullrequest.PullRequest, pullrequest.Commit))
                 {
-                    PullRequestDetails pr = statsDb.AddDataToPullRequest(pullrequest);
-                    Console.WriteLine($"{pr.CountReturned} of {pr.CountTotal} completed.");
-                    if (pr.CountReturned == pr.CountTotal)
+                    try
                     {
-                        string url = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
-                        Task<string> response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={pr.PullRequest}&commitId={pr.Commit}");
-                        response.Wait();
+                        PullRequestDetails pr = statsDb.AddDataToPullRequest(pullrequest);
+                    }
+                    catch (Exception ex)
+                    {
+                        return BadRequest(ex.ToString());
                     }
                 }
-                catch (Exception ex)
+                else
                 {
-                    return BadRequest(ex.ToString());
+                    PullRequestDetails pr = statsDb.GetPullRequest(pullrequest.PullRequest);
+                    if (pr == null)
+                        return BadRequest($"A PR with {pullrequest.PullRequest} does not exist in the database");
+                    else
+                        return BadRequest($"A PR with {pullrequest.PullRequest} does exist, but has commit number {pr.Commit}, and you submitted a commit of {pullrequest.Commit}");
                 }
             }
-            else
-            {
-                PullRequestDetails pr = statsDb.GetPullRequest(pullrequest.PullRequest);
-                if (pr == null)
-                    return BadRequest($"A PR with {pullrequest.PullRequest} does not exist in the database");
-                else
-                    return BadRequest($"A PR with {pullrequest.PullRequest} does exist, but has commit number {pr.Commit}, and you submitted a commit of {pullrequest.Commit}");
-            }
-
             return Ok();
         }
     }
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 6995ef9..7eb9d03 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -3,7 +3,6 @@
 using System;
 using System.Linq;
 using System.Threading;
-using System.Threading.Tasks;
 
 namespace APSIM.POStats.Shared
 {
@@ -47,7 +46,6 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
             pr.Commit = commitNumber;
             pr.Author = author;
             pr.CountTotal = count;
-            pr.CountReturned = 0;
             pr.Output = "";
 
             pr.Files ??= new();
@@ -74,8 +72,6 @@ public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullReques
             pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == fromPullRequest.PullRequest)
                     ?? throw new Exception($"Cannot find POStats pull request number: {fromPullRequest.PullRequest}");
 
-            pr.CountReturned += 1;
-
             foreach (ApsimFile file in fromPullRequest.Files)
                 Console.WriteLine($"File \"{file.Name}\" added to PR {fromPullRequest.PullRequest}");
 
@@ -84,7 +80,6 @@ public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullReques
 
             pr.Output += fromPullRequest.Output;
 
-            // SaveChangesMultipleTries();
             SaveChanges();
 
             return pr;
diff --git a/APSIM.POStats.Shared/TimeoutTimer.cs b/APSIM.POStats.Shared/TimeoutTimer.cs
index 05b1fdd..4fb1df8 100644
--- a/APSIM.POStats.Shared/TimeoutTimer.cs
+++ b/APSIM.POStats.Shared/TimeoutTimer.cs
@@ -1,11 +1,16 @@
+using System;
+
 namespace APSIM.POStats.Shared
 {
-    public class TimeoutTimer : System.Timers.Timer
+    public class PullRequestTimer : System.Timers.Timer
     {
         /// <summary>Pull Request Number to be closed when the timeout is reached</summary>
         public int PullRequestNumber;
 
         /// <summary>Commit id to be checked when the timeout is reached</summary>
         public string CommitId;
+
+        /// <summary>Start Time</summary>
+        public DateTime StartTime;
     }
 }
\ No newline at end of file

From 7ef83d318037b24c7c330d4ebe7a2a3cd25bbf0b Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Tue, 26 Aug 2025 14:35:17 +1000
Subject: [PATCH 12/48] Lock checks

---
 APSIM.POStats.Portal/Controllers/Api.cs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index 0b4d098..f31186d 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -13,13 +13,13 @@ namespace APSIM.POStats.Portal.Controllers
     public class Api : ControllerBase
     {
         /// <summary>The database context.</summary>
-        private readonly StatsDbContext statsDb;
+        private static StatsDbContext statsDb;
 
         /// <summary>Lock for adding to the db</summary>
-        private readonly object _lock = new();
+        private static object _lock = new();
 
         /// <summary>Event handler for finish timer.</summary>
-        private void OnCheckIfFinished(Object source, ElapsedEventArgs e)
+        private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
         {
             Console.WriteLine($"Tick");
             PullRequestTimer timer = source as PullRequestTimer;
@@ -54,7 +54,10 @@ private void OnCheckIfFinished(Object source, ElapsedEventArgs e)
         /// <param name="stats">The database context.</param>
         public Api(StatsDbContext stats)
         {
-            statsDb = stats;
+            lock (_lock)
+            {
+                statsDb = stats;
+            }
         }
 
         /// <summary>Invoked by collector to open a pull request.</summary>

From b43f8739f9b48e3904c2093d070d76c99e0dd5f5 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Thu, 4 Sep 2025 08:45:39 +1000
Subject: [PATCH 13/48] Single timer

---
 APSIM.POStats.Portal/Controllers/Api.cs       | 85 +++++++++++++------
 APSIM.POStats.Shared/GitHub/GitHub.cs         |  4 +-
 .../{TimeoutTimer.cs => PullRequestTimer.cs}  |  0
 APSIM.POStats.Shared/StatsDbContext.cs        | 27 +++++-
 4 files changed, 87 insertions(+), 29 deletions(-)
 rename APSIM.POStats.Shared/{TimeoutTimer.cs => PullRequestTimer.cs} (100%)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index f31186d..33abe01 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -13,7 +13,7 @@ namespace APSIM.POStats.Portal.Controllers
     public class Api : ControllerBase
     {
         /// <summary>The database context.</summary>
-        private static StatsDbContext statsDb;
+        private readonly StatsDbContext statsDb;
 
         /// <summary>Lock for adding to the db</summary>
         private static object _lock = new();
@@ -21,31 +21,31 @@ public class Api : ControllerBase
         /// <summary>Event handler for finish timer.</summary>
         private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
         {
+            string url = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
+            if (string.IsNullOrEmpty(url))
+                throw new Exception($"Cannot find environment variable POSTATS_UPLOAD_URL");
+
             Console.WriteLine($"Tick");
             PullRequestTimer timer = source as PullRequestTimer;
-            timer.Stop();
 
-            lock (_lock)
+            Task<string> response = WebUtilities.GetAsync($"{url}api/count?pullrequestnumber={timer.PullRequestNumber}&commitid={timer.CommitId}");
+            response.Wait();
+
+            string result = response.Result.ToString();
+            if (!string.IsNullOrEmpty(result))
             {
-                PullRequestDetails pr = statsDb.GetPullRequest(timer.PullRequestNumber);
+                int count = int.Parse(result);
+
+                Console.WriteLine($"minutes");
                 double minutes = (DateTime.Now - timer.StartTime).TotalMinutes;
-                Console.WriteLine($"{pr.PullRequest} ({pr.Commit}): {pr.Files.Count} of {pr.CountTotal} completed in {Math.Round(minutes, 1)} minutes");
+                Console.WriteLine($"{timer.PullRequestNumber} ({timer.CommitId}): running for {Math.Round(minutes, 1)} minutes");
 
-                if (pr.Commit == timer.CommitId)
+                Console.WriteLine($"{count} == 0 || {minutes} >= 20");
+                if (count == 0 || minutes >= 20)
                 {
-                    if (pr.Files.Count >= pr.CountTotal || minutes >= 40)
-                    {
-                        string url = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
-                        if (string.IsNullOrEmpty(url))
-                            throw new Exception($"Cannot find environment variable POSTATS_UPLOAD_URL");
-
-                        Task<string> response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequestNumber}&commitId={timer.CommitId}");
-                        response.Wait();
-                    }
-                    else
-                    {
-                        timer.Start();
-                    }
+                    response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequestNumber}&commitId={timer.CommitId}");
+                    response.Wait();
+                    timer.Stop();
                 }
             }
         }
@@ -54,10 +54,7 @@ private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
         /// <param name="stats">The database context.</param>
         public Api(StatsDbContext stats)
         {
-            lock (_lock)
-            {
-                statsDb = stats;
-            }
+            statsDb = stats;
         }
 
         /// <summary>Invoked by collector to open a pull request.</summary>
@@ -81,8 +78,8 @@ public IActionResult Open(int pullrequestnumber, string commitid, int count, str
                 statsDb.OpenPullRequest(pullrequestnumber, commitid, author, count);
 
                 // Create a timer to check how many files have been returned
-                PullRequestTimer finishTimer = new PullRequestTimer {Interval=10000, PullRequestNumber=pullrequestnumber, CommitId=commitid};
-                finishTimer.Elapsed += new ElapsedEventHandler(OnCheckIfFinished);
+                PullRequestTimer finishTimer = new PullRequestTimer { Interval = 10000, PullRequestNumber = pullrequestnumber, CommitId = commitid };
+                finishTimer.Elapsed += OnCheckIfFinished;
                 finishTimer.AutoReset = true;
                 finishTimer.StartTime = DateTime.Now;
                 finishTimer.Start();
@@ -138,6 +135,44 @@ public IActionResult Close(int pullrequestnumber, string commitid)
             return Ok();
         }
 
+        /// <summary>Returns the number of files remaining for a pull request</summary>
+        /// <param name="pullrequestnumber">The number of the pull request.</param>
+        /// <param name="pullrequestnumber">The commit id of the pull request.</param>
+        /// <returns></returns>
+        [HttpGet("count")]
+        public IActionResult Count(int pullrequestnumber, string commitid)
+        {
+            Console.WriteLine($"api/count");
+            
+            if (pullrequestnumber == 0)
+                return BadRequest("You must supply a pull request number");
+
+            if (commitid.Length == 0)
+                return BadRequest("You must supply a commit number");
+
+            if (statsDb.PullRequestWithCommitExists(pullrequestnumber, commitid))
+            {
+                try
+                {
+                    var count = statsDb.GetNumberOfFilesInPullRequestRemaining(pullrequestnumber, commitid);
+                    Console.WriteLine($"{count}");
+                    return Ok(count);
+                }
+                catch (Exception ex)
+                {
+                    return BadRequest(ex.ToString());
+                }
+            }
+            else
+            {
+                PullRequestDetails pr = statsDb.GetPullRequest(pullrequestnumber);
+                if (pr == null)
+                    return BadRequest($"A PR with {pullrequestnumber} does not exist in the database");
+                else
+                    return BadRequest($"A PR with {pullrequestnumber} does exist, but has commit number {pr.Commit}, and you submitted a commit of {commitid}");
+            }
+        }
+
         /// <summary>Invoked by collector to upload a pull request.</summary>
         /// <param name="pullRequest"></param>
         /// <returns></returns>
diff --git a/APSIM.POStats.Shared/GitHub/GitHub.cs b/APSIM.POStats.Shared/GitHub/GitHub.cs
index 574523d..2491fc0 100644
--- a/APSIM.POStats.Shared/GitHub/GitHub.cs
+++ b/APSIM.POStats.Shared/GitHub/GitHub.cs
@@ -69,9 +69,9 @@ public static void SetStatus(int pullRequestNumber, string commitId, VariableCom
                 state = "success";
 
             if (status == VariableComparison.Status.Running)
-                state = "pending";
+                state = "success"; //state = "pending";
 
-            string stateFormatted = status.ToString();
+            string stateFormatted = "In Development: " + status.ToString();
             if (!String.IsNullOrEmpty(message))
                 stateFormatted = message;
 
diff --git a/APSIM.POStats.Shared/TimeoutTimer.cs b/APSIM.POStats.Shared/PullRequestTimer.cs
similarity index 100%
rename from APSIM.POStats.Shared/TimeoutTimer.cs
rename to APSIM.POStats.Shared/PullRequestTimer.cs
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 7eb9d03..9e8f8f9 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -69,8 +69,9 @@ public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullReques
             var pr = new PullRequestDetails();
 
             // Find the pull request. Should always exist if OpenPullRequest has been called.
-            pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == fromPullRequest.PullRequest)
-                    ?? throw new Exception($"Cannot find POStats pull request number: {fromPullRequest.PullRequest}");
+            pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == fromPullRequest.PullRequest && pr.Commit == fromPullRequest.Commit);
+            if (pr == null)
+                throw new Exception($"Cannot find POStats pull request number: {fromPullRequest.PullRequest}");
 
             foreach (ApsimFile file in fromPullRequest.Files)
                 Console.WriteLine($"File \"{file.Name}\" added to PR {fromPullRequest.PullRequest}");
@@ -80,11 +81,33 @@ public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullReques
 
             pr.Output += fromPullRequest.Output;
 
+            pr.CountReturned += 1;
+
             SaveChanges();
 
             return pr;
         }
 
+        /// <summary>
+        /// Add file data to a pull request.
+        /// </summary>
+        /// <remarks>
+        /// Use case: A collector calls this method to add data to a pull request.
+        /// </remarks>
+        /// <param name="pullRequest">The pull request to copy the data from..</param>
+        /// <returns>Reference to stored PullRequest</returns>
+        public int GetNumberOfFilesInPullRequestRemaining(int pullrequestnumber, string commitid)
+        {
+            var pr = new PullRequestDetails();
+
+            // Find the pull request. Should always exist if OpenPullRequest has been called.
+            pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == pullrequestnumber && pr.Commit == commitid);
+            if (pr == null)
+                throw new Exception($"Cannot find POStats pull request number: {pullrequestnumber}");
+
+            return pr.CountTotal - pr.CountReturned;
+        }
+
         public bool SaveChangesMultipleTries(int retries = 0)
         {
             try

From 560434adb856d022c54489eee302505fb4b360a4 Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Thu, 4 Sep 2025 10:33:20 +1000
Subject: [PATCH 14/48] Add NodeStatuses list to track how many nodes have
 finished successfully

---
 APSIM.POStats.Shared/Models/PullRequestDetails.cs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/APSIM.POStats.Shared/Models/PullRequestDetails.cs b/APSIM.POStats.Shared/Models/PullRequestDetails.cs
index d51638b..9cbdd1a 100644
--- a/APSIM.POStats.Shared/Models/PullRequestDetails.cs
+++ b/APSIM.POStats.Shared/Models/PullRequestDetails.cs
@@ -17,5 +17,8 @@ public class PullRequestDetails
         public string Output { get; set; }
         public int? AcceptedPullRequestId { get; set; }
         public virtual PullRequestDetails AcceptedPullRequest { get; set; }
+
+        /// <summary>Statuses of the nodes that ran the tests. True = success, false = failure.</summary>
+        public virtual List<bool> NodeStatuses { get; set; }
     }
 }
\ No newline at end of file

From add1e02f49f0f63d44c859874f4d6009774c47f9 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Wed, 10 Sep 2025 10:29:10 +1000
Subject: [PATCH 15/48] Lots of changes in an attempt to track job finishes

---
 APSIM.POStats.Portal/Controllers/Api.cs       | 16 +++++------
 APSIM.POStats.Portal/Pages/Index.cshtml       | 11 ++++++--
 APSIM.POStats.Shared/Collector.cs             | 17 +++++++++---
 APSIM.POStats.Shared/GitHub/GitHub.cs         |  5 +++-
 .../Models/PullRequestDetails.cs              | 16 +++++++----
 APSIM.POStats.Shared/PullRequestTimer.cs      |  4 +--
 APSIM.POStats.Shared/StatsDbContext.cs        | 27 +++++++++++++------
 7 files changed, 65 insertions(+), 31 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index 33abe01..95fedcd 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -28,7 +28,7 @@ private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
             Console.WriteLine($"Tick");
             PullRequestTimer timer = source as PullRequestTimer;
 
-            Task<string> response = WebUtilities.GetAsync($"{url}api/count?pullrequestnumber={timer.PullRequestNumber}&commitid={timer.CommitId}");
+            Task<string> response = WebUtilities.GetAsync($"{url}api/count?pullrequestnumber={timer.PullRequest}&commitid={timer.Commit}");
             response.Wait();
 
             string result = response.Result.ToString();
@@ -36,16 +36,14 @@ private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
             {
                 int count = int.Parse(result);
 
-                Console.WriteLine($"minutes");
                 double minutes = (DateTime.Now - timer.StartTime).TotalMinutes;
-                Console.WriteLine($"{timer.PullRequestNumber} ({timer.CommitId}): running for {Math.Round(minutes, 1)} minutes");
 
-                Console.WriteLine($"{count} == 0 || {minutes} >= 20");
-                if (count == 0 || minutes >= 20)
+                Console.WriteLine($"{timer.PullRequest} ({timer.Commit}): count={count} and minutes={Math.Round(minutes, 1)}");
+                if (minutes >= 20)
                 {
-                    response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequestNumber}&commitId={timer.CommitId}");
-                    response.Wait();
                     timer.Stop();
+                    response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequest}&commitId={timer.Commit}");
+                    response.Wait();
                 }
             }
         }
@@ -78,7 +76,7 @@ public IActionResult Open(int pullrequestnumber, string commitid, int count, str
                 statsDb.OpenPullRequest(pullrequestnumber, commitid, author, count);
 
                 // Create a timer to check how many files have been returned
-                PullRequestTimer finishTimer = new PullRequestTimer { Interval = 10000, PullRequestNumber = pullrequestnumber, CommitId = commitid };
+                PullRequestTimer finishTimer = new PullRequestTimer { Interval = 10000, PullRequest = pullrequestnumber, Commit = commitid };
                 finishTimer.Elapsed += OnCheckIfFinished;
                 finishTimer.AutoReset = true;
                 finishTimer.StartTime = DateTime.Now;
@@ -154,7 +152,7 @@ public IActionResult Count(int pullrequestnumber, string commitid)
             {
                 try
                 {
-                    var count = statsDb.GetNumberOfFilesInPullRequestRemaining(pullrequestnumber, commitid);
+                    var count = statsDb.GetNumberOfCompletesInPullRequest(pullrequestnumber, commitid);
                     Console.WriteLine($"{count}");
                     return Ok(count);
                 }
diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml b/APSIM.POStats.Portal/Pages/Index.cshtml
index ea2cbde..37e0a05 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml
@@ -80,6 +80,10 @@
                 <label for="AcceptedPullRequestNumber">Accepted pull request number:</label>
                 <input name="AcceptedPullRequestNumber" value="@Model.PullRequest.AcceptedPullRequest?.PullRequest" readonly />
             </div>
+            <div>
+                <label for="ReturnCount">Return Count:</label>
+                <input name="ReturnCount" value="@Model.PullRequest.Status.Count" readonly />
+            </div>
             <br />
             <br />
             <div>
@@ -188,9 +192,12 @@
             Logs:
             @if (Model.ShowLogs) 
             {
-                @foreach (string line in @Model.PullRequest.Output.Split("\n"))
+                @foreach (string log in Model.PullRequest.Outputs)
                 {
-                    @Html.Encode(line)<br />
+                    @foreach (string line in log.Split("\n"))
+                    {
+                        @Html.Encode(line)<br />
+                    }
                 }
             }
             else
diff --git a/APSIM.POStats.Shared/Collector.cs b/APSIM.POStats.Shared/Collector.cs
index f915af6..457b94a 100644
--- a/APSIM.POStats.Shared/Collector.cs
+++ b/APSIM.POStats.Shared/Collector.cs
@@ -42,7 +42,8 @@ public static PullRequestDetails RetrieveData(int pullId, string commitId, strin
             };
 
             string errorMessages = string.Empty;
-            pullRequest.Output = "";
+            pullRequest.Outputs = new List<string>();
+            pullRequest.Status = new List<bool>();
             foreach (var filePath in filePaths)
             {
                 try
@@ -63,10 +64,11 @@ public static PullRequestDetails RetrieveData(int pullId, string commitId, strin
                 {
                     errorMessages += ex.ToString() + "\n";
                 }
-                
+
             }
 
             FileInfo[] files = new DirectoryInfo("./").GetFiles("*stdout.txt", SearchOption.AllDirectories);
+            string log = "";
             foreach (FileInfo fileInfo in files)
             {
                 Console.WriteLine(fileInfo.FullName);
@@ -75,17 +77,24 @@ public static PullRequestDetails RetrieveData(int pullId, string commitId, strin
                     string text = "";
                     while ((text = sr.ReadLine()) != null)
                     {
-                        pullRequest.Output += text + "\n";
+                        log += text + "\n";
                     }
                 }
             }
 
             if (errorMessages.Length > 0)
             {
-                pullRequest.Output += errorMessages;
+                log += errorMessages;
                 pullRequest.Files.Clear();
+                pullRequest.Status.Add(false);
+            }
+            else
+            {
+                pullRequest.Status.Add(true);
             }
 
+            pullRequest.Outputs.Add(log);
+            
             return pullRequest;
         }
 
diff --git a/APSIM.POStats.Shared/GitHub/GitHub.cs b/APSIM.POStats.Shared/GitHub/GitHub.cs
index 2491fc0..18bb73a 100644
--- a/APSIM.POStats.Shared/GitHub/GitHub.cs
+++ b/APSIM.POStats.Shared/GitHub/GitHub.cs
@@ -69,7 +69,7 @@ public static void SetStatus(int pullRequestNumber, string commitId, VariableCom
                 state = "success";
 
             if (status == VariableComparison.Status.Running)
-                state = "success"; //state = "pending";
+                state = "pending";
 
             string stateFormatted = "In Development: " + status.ToString();
             if (!String.IsNullOrEmpty(message))
@@ -81,6 +81,9 @@ public static void SetStatus(int pullRequestNumber, string commitId, VariableCom
 
             Console.WriteLine($"Github Status Update: {state} {urlStr} {stateFormatted} {"APSIM.POStats2"}");
 
+            //Always success for now
+            state = "success";
+
             //Status POST body details
             GitHubStatusDetails body = new GitHubStatusDetails(state, urlStr, stateFormatted, "APSIM.POStats2");
 
diff --git a/APSIM.POStats.Shared/Models/PullRequestDetails.cs b/APSIM.POStats.Shared/Models/PullRequestDetails.cs
index 9cbdd1a..c1efd72 100644
--- a/APSIM.POStats.Shared/Models/PullRequestDetails.cs
+++ b/APSIM.POStats.Shared/Models/PullRequestDetails.cs
@@ -10,15 +10,21 @@ public class PullRequestDetails
         public string Commit { get; set; }
         public string Author { get; set; }
         public DateTime DateRun { get; set; }
-        public DateTime? DateStatsAccepted { get; set; }
-        public virtual List<ApsimFile> Files { get; set; }
+
+        /// <summary>Number of nodes that should return</summary>
         public int CountTotal { get; set; }
-        public int CountReturned { get; set; }
-        public string Output { get; set; }
+
+        public DateTime? DateStatsAccepted { get; set; }
         public int? AcceptedPullRequestId { get; set; }
+
         public virtual PullRequestDetails AcceptedPullRequest { get; set; }
 
+        public virtual List<ApsimFile> Files { get; set; }
+
         /// <summary>Statuses of the nodes that ran the tests. True = success, false = failure.</summary>
-        public virtual List<bool> NodeStatuses { get; set; }
+        public virtual List<bool> Status { get; set; }
+
+        /// <summary>Log outputs from validation that are returned</summary>
+        public virtual List<string> Outputs { get; set; }
     }
 }
\ No newline at end of file
diff --git a/APSIM.POStats.Shared/PullRequestTimer.cs b/APSIM.POStats.Shared/PullRequestTimer.cs
index 4fb1df8..413f864 100644
--- a/APSIM.POStats.Shared/PullRequestTimer.cs
+++ b/APSIM.POStats.Shared/PullRequestTimer.cs
@@ -5,10 +5,10 @@ namespace APSIM.POStats.Shared
     public class PullRequestTimer : System.Timers.Timer
     {
         /// <summary>Pull Request Number to be closed when the timeout is reached</summary>
-        public int PullRequestNumber;
+        public int PullRequest;
 
         /// <summary>Commit id to be checked when the timeout is reached</summary>
-        public string CommitId;
+        public string Commit;
 
         /// <summary>Start Time</summary>
         public DateTime StartTime;
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 9e8f8f9..5e0aa4f 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -1,4 +1,5 @@
 using APSIM.POStats.Shared.Models;
+using Microsoft.CodeAnalysis.CSharp.Syntax;
 using Microsoft.EntityFrameworkCore;
 using System;
 using System.Linq;
@@ -45,14 +46,22 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
             }
             pr.Commit = commitNumber;
             pr.Author = author;
+            pr.DateRun = DateTime.Now;
             pr.CountTotal = count;
-            pr.Output = "";
 
-            pr.Files ??= new();
-            pr.Files.Clear();
-            pr.DateRun = DateTime.Now;
             pr.DateStatsAccepted = null;
+            pr.AcceptedPullRequestId = null;
             pr.AcceptedPullRequest = null;
+
+            pr.Files ??= new();
+            pr.Files.Clear();
+
+            pr.Outputs ??= new();
+            pr.Outputs.Clear();
+
+            pr.Status ??= new();
+            pr.Status.Clear();
+
             SaveChanges();
         }
 
@@ -79,9 +88,11 @@ public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullReques
             if (fromPullRequest.Files != null)
                 pr.Files.AddRange(fromPullRequest.Files);
 
-            pr.Output += fromPullRequest.Output;
+            if (fromPullRequest.Outputs != null)
+                pr.Outputs.AddRange(fromPullRequest.Outputs);
 
-            pr.CountReturned += 1;
+            if (fromPullRequest.Status != null)
+                pr.Status.AddRange(fromPullRequest.Status);
 
             SaveChanges();
 
@@ -96,7 +107,7 @@ public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullReques
         /// </remarks>
         /// <param name="pullRequest">The pull request to copy the data from..</param>
         /// <returns>Reference to stored PullRequest</returns>
-        public int GetNumberOfFilesInPullRequestRemaining(int pullrequestnumber, string commitid)
+        public int GetNumberOfCompletesInPullRequest(int pullrequestnumber, string commitid)
         {
             var pr = new PullRequestDetails();
 
@@ -105,7 +116,7 @@ public int GetNumberOfFilesInPullRequestRemaining(int pullrequestnumber, string
             if (pr == null)
                 throw new Exception($"Cannot find POStats pull request number: {pullrequestnumber}");
 
-            return pr.CountTotal - pr.CountReturned;
+            return pr.Status.Count;
         }
 
         public bool SaveChangesMultipleTries(int retries = 0)

From b38b1961c4e499b4f57ef2da6294977de29e03b9 Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Wed, 10 Sep 2025 12:02:55 +1000
Subject: [PATCH 16/48] add pool member to PullRequestDetails

---
 APSIM.POStats.Shared/Models/PullRequestDetails.cs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/APSIM.POStats.Shared/Models/PullRequestDetails.cs b/APSIM.POStats.Shared/Models/PullRequestDetails.cs
index c1efd72..1566f59 100644
--- a/APSIM.POStats.Shared/Models/PullRequestDetails.cs
+++ b/APSIM.POStats.Shared/Models/PullRequestDetails.cs
@@ -26,5 +26,8 @@ public class PullRequestDetails
 
         /// <summary>Log outputs from validation that are returned</summary>
         public virtual List<string> Outputs { get; set; }
+
+        /// <summary>The name of the Azure Batch pool that was used to run tests for this pull request.</summary>
+        public string Pool { get; set; }
     }
 }
\ No newline at end of file

From 9641edadd8bdf54bb18beb1a35de17ae3841a524 Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Wed, 10 Sep 2025 12:03:17 +1000
Subject: [PATCH 17/48] Add Azure batch package to allow batch resource
 handling

---
 APSIM.POStats.Portal/APSIM.POStats.Portal.csproj | 1 +
 1 file changed, 1 insertion(+)

diff --git a/APSIM.POStats.Portal/APSIM.POStats.Portal.csproj b/APSIM.POStats.Portal/APSIM.POStats.Portal.csproj
index 3d040b0..c1729bf 100644
--- a/APSIM.POStats.Portal/APSIM.POStats.Portal.csproj
+++ b/APSIM.POStats.Portal/APSIM.POStats.Portal.csproj
@@ -21,6 +21,7 @@
     </PackageReference>
     <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.0" />
     <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="8.0.3" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="16.3.1" />
   </ItemGroup>
 
   <ItemGroup>

From aa4575dd4f04c7fce116f12e1e9c5b46f8f9657d Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Wed, 10 Sep 2025 12:04:03 +1000
Subject: [PATCH 18/48] Create AzureBatchManager class for handling batch
 related functions

---
 APSIM.POStats.Portal/Models/Azure.cs | 72 ++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 APSIM.POStats.Portal/Models/Azure.cs

diff --git a/APSIM.POStats.Portal/Models/Azure.cs b/APSIM.POStats.Portal/Models/Azure.cs
new file mode 100644
index 0000000..f202a22
--- /dev/null
+++ b/APSIM.POStats.Portal/Models/Azure.cs
@@ -0,0 +1,72 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Azure.Batch;
+using Microsoft.Azure.Batch.Auth;
+using Microsoft.Azure.Batch.Common;
+namespace APSIM.POStats.Portal.Models
+{
+    /// <summary>
+    /// Class for managing Azure Batch resources.
+    /// </summary>
+    public class AzureBatchManager
+    {
+
+        /// <summary>
+        /// Closes (deletes) a specified Azure Batch pool.
+        /// </summary>
+        /// <param name="resourceGroupName">The name of the Azure resource group.</param>
+        /// <param name="accountName">The name of the Azure Batch account.</param>
+        /// <param name="poolName">The name of the Azure Batch pool to delete.</param>
+        public static async Task CloseBatchPoolAsync(string poolName)
+        {
+            try
+            {
+                using BatchClient _batchClient = GetBatchClient();
+
+                if (_batchClient == null)
+                    throw new InvalidOperationException("Unable to get a Azure batch client while closing a pool.");
+
+                var pools = await _batchClient.PoolOperations.ListPools().ToListAsync();
+                CloudPool pool = pools.FirstOrDefault(p => p.Id == poolName);
+
+                await _batchClient.PoolOperations.DeletePoolAsync(pool.Id);
+            }
+            catch (BatchException be)
+            {
+                Console.WriteLine(be.ToString());
+                throw;
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine(e.ToString());
+                throw;
+            }
+        }
+
+        /// <summary>
+        /// Get a BatchClient object to interact with Azure Batch service.
+        /// </summary>
+        /// <returns>A <c>BatchClient</c> object for interacting with Azure Batch service.</returns>
+        private static BatchClient GetBatchClient()
+        {
+            // Ensure the necessary environment variables exist.
+            string accountUrl = Environment.GetEnvironmentVariable("AZURE_BATCH_ACCOUNT_URL");
+            if (string.IsNullOrEmpty(accountUrl))
+                throw new Exception("Cannot find variable AZURE_BATCH_ACCOUNT_URL");
+
+            string accountName = Environment.GetEnvironmentVariable("AZURE_BATCH_ACCOUNT_NAME");
+            if (string.IsNullOrEmpty(accountName))
+                throw new Exception("Cannot find variable AZURE_BATCH_ACCOUNT_NAME");
+
+            string primaryAccessKey = Environment.GetEnvironmentVariable("AZURE_BATCH_ACCOUNT_KEY");
+            if (string.IsNullOrEmpty(primaryAccessKey))
+                throw new Exception("Cannot find variable AZURE_BATCH_ACCOUNT_KEY");
+
+            return BatchClient.Open(new BatchSharedKeyCredentials(
+                Environment.GetEnvironmentVariable("AZURE_BATCH_ACCOUNT_URL"),
+                Environment.GetEnvironmentVariable("AZURE_BATCH_ACCOUNT_NAME"),
+                Environment.GetEnvironmentVariable("AZURE_BATCH_ACCOUNT_KEY")));
+        }
+    }
+}
\ No newline at end of file

From 97919ba2631b07077ae47f3bcbe4cf056852c2cf Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Wed, 10 Sep 2025 12:04:38 +1000
Subject: [PATCH 19/48] When closing a PR make it also close the PR's Azure
 Batch pool

---
 APSIM.POStats.Portal/Controllers/Api.cs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index 95fedcd..efa25d3 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -5,6 +5,7 @@
 using APSIM.POStats.Shared.GitHub;
 using Microsoft.AspNetCore.Mvc;
 using System.Threading.Tasks;
+using APSIM.POStats.Portal.Models;
 
 namespace APSIM.POStats.Portal.Controllers
 {
@@ -94,7 +95,7 @@ public IActionResult Open(int pullrequestnumber, string commitid, int count, str
         /// <param name="pullrequestnumber">The number of the pull request to close.</param>
         /// <returns></returns>
         [HttpGet("close")]
-        public IActionResult Close(int pullrequestnumber, string commitid)
+        public async Task<IActionResult> Close(int pullrequestnumber, string commitid)
         {
             Console.WriteLine($"api/close called");
             
@@ -115,6 +116,7 @@ public IActionResult Close(int pullrequestnumber, string commitid)
 
                     VariableComparison.Status status = PullRequestFunctions.GetStatus(pullRequest);
                     GitHub.SetStatus(pullrequestnumber, commitid, status);
+                    await AzureBatchManager.CloseBatchPoolAsync(pullRequest.Pool);
                 }
                 catch (Exception ex)
                 {

From e9cdb75b7e883e1ffc38dbcecc6e789f7cf15f95 Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Wed, 10 Sep 2025 14:03:22 +1000
Subject: [PATCH 20/48] Change Collector to accept a pool value used with
 github workflow for validation

---
 APSIM.POStats.Collector/Program.cs | 11 ++++++++---
 APSIM.POStats.Shared/Collector.cs  |  5 +++--
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/APSIM.POStats.Collector/Program.cs b/APSIM.POStats.Collector/Program.cs
index 2c7ad0c..731c04d 100644
--- a/APSIM.POStats.Collector/Program.cs
+++ b/APSIM.POStats.Collector/Program.cs
@@ -27,7 +27,8 @@ static int Main(string[] args)
                 Console.WriteLine("  3. (int) Commit Id");
                 Console.WriteLine("  4. (string) UserID");
                 Console.WriteLine("  5. (datetime) Date");
-                Console.WriteLine("  6. (string) Directories (space separated)");
+                Console.WriteLine("  6. (string) Azure pool");
+                Console.WriteLine("  7. (string) Directories (space separated)");
                 Console.WriteLine(@"  Example: APSIM.POStats.Collector Upload 1111 abcdef12345 2016.12.01-06:33 hol353 c:\Apsimx\Tests c:\Apsimx\UnderReview");
                 return 1;
             }
@@ -52,9 +53,12 @@ static int Main(string[] args)
                 //get the run date
                 DateTime runDate = DateTime.ParseExact(args[4], "yyyy.M.d-HH:mm", CultureInfo.CurrentCulture, DateTimeStyles.AssumeLocal);
 
+                //get the pool name
+                string pool = args[5];
+
                 //get the file paths
                 List<string> filePaths = new List<string>();
-                for (int i = 5; i < args.Length; i++)
+                for (int i = 6; i < args.Length; i++)
                 {
                     if (File.Exists(args[i]))
                     {
@@ -64,8 +68,9 @@ static int Main(string[] args)
                         throw new Exception($"File \"{args[i]}\" does not exist.");
                 }
 
+
                 //get the Pull Request details
-                PullRequestDetails pullRequest = Shared.Collector.RetrieveData(pullId, commitId, author, runDate, filePaths);
+                PullRequestDetails pullRequest = Shared.Collector.RetrieveData(pullId, commitId, author, runDate, pool, filePaths);
 
                 string url = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
                 Console.WriteLine($"{url}");
diff --git a/APSIM.POStats.Shared/Collector.cs b/APSIM.POStats.Shared/Collector.cs
index 457b94a..e26712a 100644
--- a/APSIM.POStats.Shared/Collector.cs
+++ b/APSIM.POStats.Shared/Collector.cs
@@ -30,7 +30,7 @@ public class Collector
         /// <param name="author"></param>
         /// <param name="runDate"></param>
         /// <param name="filePaths">A collection of file paths to search.</param>
-        public static PullRequestDetails RetrieveData(int pullId, string commitId, string author, DateTime runDate, IEnumerable<string> filePaths)
+        public static PullRequestDetails RetrieveData(int pullId, string commitId, string author, DateTime runDate, string pool, IEnumerable<string> filePaths)
         {
             var pullRequest = new PullRequestDetails()
             {
@@ -38,7 +38,8 @@ public static PullRequestDetails RetrieveData(int pullId, string commitId, strin
                 Commit = commitId,
                 Author = author,
                 DateRun = runDate,
-                Files = new List<ApsimFile>()
+                Files = new List<ApsimFile>(),
+                Pool = pool
             };
 
             string errorMessages = string.Empty;

From 2670ccf663135833e9d1b250a8ba7693725e763e Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Wed, 10 Sep 2025 14:39:34 +1000
Subject: [PATCH 21/48] Fix Collector tests where RetrieveData method was
 missing an argument

---
 APSIM.POStats.Tests/CollectorTests.cs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/APSIM.POStats.Tests/CollectorTests.cs b/APSIM.POStats.Tests/CollectorTests.cs
index 276d0cc..853ffb4 100644
--- a/APSIM.POStats.Tests/CollectorTests.cs
+++ b/APSIM.POStats.Tests/CollectorTests.cs
@@ -66,7 +66,7 @@ public void TestEnsureNormalCollectorOperationWorks()
             " 1,2000-01-04, 200.0, 210.0" + Environment.NewLine
             ));
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), new string[] { path });
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
 
         string jsonString = JsonSerializer.Serialize(pullRequest);;
 
@@ -123,7 +123,7 @@ public void EnsurePredictedStringColumnsAreIgnored()
             ));
         database.Close();
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), new string[] { path });
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
         Assert.That(pullRequest.Files.ToList().Count, Is.EqualTo(0));
     }
 
@@ -142,7 +142,7 @@ public void EnsureObservedStringColumnsAreIgnored()
             ));
         database.Close();
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), new string[] { path });
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
         Assert.That(pullRequest.Files.ToList().Count, Is.EqualTo(0));
     }
 
@@ -161,7 +161,7 @@ public void EnsureObservedStringValuesInRowsAreIgnored()
             ));
         database.Close();
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), new string[] { path });
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
         Assert.That(pullRequest.Files.ToList().Count, Is.EqualTo(1));
         Assert.That(pullRequest.Files[0].Tables.Count, Is.EqualTo(1));
 
@@ -189,7 +189,7 @@ public void EnsurePOTableNotUnderDataStoreIsFound()
             ));
         database.Close();
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), new string[] { path });
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
         Assert.That(pullRequest.Files.ToList().Count, Is.EqualTo(1));
         Assert.That(pullRequest.Files[0].Tables.Count, Is.EqualTo(1));
 
@@ -216,7 +216,7 @@ public void EnsureCollectorFindsIntegers()
             ));
         database.Close();
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), new string[] { path });
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
         Assert.That(pullRequest.Files.ToList().Count, Is.EqualTo(1));
         Assert.That(pullRequest.Files[0].Tables.Count, Is.EqualTo(1));
 

From 60a4e77e1d30b8d9e20f38e24d486958a6884f0d Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Wed, 10 Sep 2025 15:13:48 +1000
Subject: [PATCH 22/48] fix unit tests

---
 APSIM.POStats.Tests/CollectorTests.cs | 24 ++++++++++++++----------
 APSIM.POStats.Tests/StatsDbTests.cs   |  1 +
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/APSIM.POStats.Tests/CollectorTests.cs b/APSIM.POStats.Tests/CollectorTests.cs
index 853ffb4..be9081c 100644
--- a/APSIM.POStats.Tests/CollectorTests.cs
+++ b/APSIM.POStats.Tests/CollectorTests.cs
@@ -12,6 +12,8 @@ public class TestsCollector
     private string path;
     private SqliteConnection database;
 
+    private string[] files;
+
     [SetUp]
     public void SetUp()
     {
@@ -27,14 +29,15 @@ public void SetUp()
 
         var filename = Path.Combine(path, "Test.apsimx");
         using (var writer = new FileStream(filename, FileMode.Create))
-            using (Stream stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("APSIM.POStats.Tests.Test.apsimx"))
-                if (stream != null)
-                    stream.CopyTo(writer);
+        using (Stream stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("APSIM.POStats.Tests.Test.apsimx"))
+            if (stream != null)
+                stream.CopyTo(writer);
 
         // Create an empty database.
         var dbFileName = Path.Combine(path, "Test.db");
         database = new SqliteConnection($"Data source={dbFileName}");
         database.Open();
+        files = Directory.GetFiles(path, "*.apsimx");
     }
 
     [TearDown]
@@ -66,9 +69,10 @@ public void TestEnsureNormalCollectorOperationWorks()
             " 1,2000-01-04, 200.0, 210.0" + Environment.NewLine
             ));
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
 
-        string jsonString = JsonSerializer.Serialize(pullRequest);;
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", files);
+
+        string jsonString = JsonSerializer.Serialize(pullRequest);
 
         Assert.That(pullRequest.Files.ToList().Count, Is.EqualTo(1));
         Assert.That(pullRequest.Files[0].Tables.Count, Is.EqualTo(2));
@@ -123,7 +127,7 @@ public void EnsurePredictedStringColumnsAreIgnored()
             ));
         database.Close();
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", files);
         Assert.That(pullRequest.Files.ToList().Count, Is.EqualTo(0));
     }
 
@@ -142,7 +146,7 @@ public void EnsureObservedStringColumnsAreIgnored()
             ));
         database.Close();
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", files);
         Assert.That(pullRequest.Files.ToList().Count, Is.EqualTo(0));
     }
 
@@ -161,7 +165,7 @@ public void EnsureObservedStringValuesInRowsAreIgnored()
             ));
         database.Close();
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", files);
         Assert.That(pullRequest.Files.ToList().Count, Is.EqualTo(1));
         Assert.That(pullRequest.Files[0].Tables.Count, Is.EqualTo(1));
 
@@ -189,7 +193,7 @@ public void EnsurePOTableNotUnderDataStoreIsFound()
             ));
         database.Close();
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", files);
         Assert.That(pullRequest.Files.ToList().Count, Is.EqualTo(1));
         Assert.That(pullRequest.Files[0].Tables.Count, Is.EqualTo(1));
 
@@ -216,7 +220,7 @@ public void EnsureCollectorFindsIntegers()
             ));
         database.Close();
 
-        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", new string[] { path });
+        var pullRequest = Collector.RetrieveData(1234, "1", null, new DateTime(2000, 1, 1), "workflo-pool", files);
         Assert.That(pullRequest.Files.ToList().Count, Is.EqualTo(1));
         Assert.That(pullRequest.Files[0].Tables.Count, Is.EqualTo(1));
 
diff --git a/APSIM.POStats.Tests/StatsDbTests.cs b/APSIM.POStats.Tests/StatsDbTests.cs
index fbc8136..1c1db48 100644
--- a/APSIM.POStats.Tests/StatsDbTests.cs
+++ b/APSIM.POStats.Tests/StatsDbTests.cs
@@ -48,6 +48,7 @@ public void TestAddDataToPullRequest()
         PullRequestDetails prToAdd = new()
         {
             PullRequest = 1234,
+            Commit = "1",
             Files = new()
             {
                 new ApsimFile()

From 4879520f748d000b0316b06b4e970cd614deabe4 Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Thu, 11 Sep 2025 14:55:23 +1000
Subject: [PATCH 23/48] Add a script that builds and pushes postats2 images

---
 build-and-push-to-dockerhub.sh | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 build-and-push-to-dockerhub.sh

diff --git a/build-and-push-to-dockerhub.sh b/build-and-push-to-dockerhub.sh
new file mode 100644
index 0000000..726db09
--- /dev/null
+++ b/build-and-push-to-dockerhub.sh
@@ -0,0 +1,17 @@
+# This script builds and pushes the Docker images to Docker Hub
+
+echo "Building and pushing Docker images to Docker Hub..."
+echo "Make sure you are logged in to Docker Hub using 'docker login' before running this script."
+echo "Also ensure that you have permission to push to the 'apsiminitiative' repository."
+echo "You can login as apsimbot if you have the credentials."
+echo "You can run './build.sh' to build the images locally before pushing."
+echo ""
+
+# build the image first
+./build.sh
+
+# push the images to Docker Hub
+docker push apsiminitiative/postats2-portal:latest
+docker push apsiminitiative/postats2-collector:latest
+
+echo "Done."
\ No newline at end of file

From 1136d100458e280047f34e59b52cd532fde5f33f Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Fri, 12 Sep 2025 15:57:37 +1000
Subject: [PATCH 24/48] Change timeout and ensure azure batch manager can close
 a pool

---
 APSIM.POStats.Portal/Controllers/Api.cs       |  5 ++-
 APSIM.POStats.Portal/Models/Azure.cs          | 13 ++++----
 .../APSIM.POStats.Tests.csproj                |  2 ++
 APSIM.POStats.Tests/AzureTests.cs             | 32 +++++++++++++++++++
 4 files changed, 44 insertions(+), 8 deletions(-)
 create mode 100644 APSIM.POStats.Tests/AzureTests.cs

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index efa25d3..e61a8e5 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -19,6 +19,9 @@ public class Api : ControllerBase
         /// <summary>Lock for adding to the db</summary>
         private static object _lock = new();
 
+        /// <summary>Final timeout in minutes. Controls how long to wait before finally closing a pull request.</summary>
+        private const double FINAL_TIMEOUT = 30.0;
+
         /// <summary>Event handler for finish timer.</summary>
         private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
         {
@@ -40,7 +43,7 @@ private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
                 double minutes = (DateTime.Now - timer.StartTime).TotalMinutes;
 
                 Console.WriteLine($"{timer.PullRequest} ({timer.Commit}): count={count} and minutes={Math.Round(minutes, 1)}");
-                if (minutes >= 20)
+                if (minutes >= FINAL_TIMEOUT)
                 {
                     timer.Stop();
                     response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequest}&commitId={timer.Commit}");
diff --git a/APSIM.POStats.Portal/Models/Azure.cs b/APSIM.POStats.Portal/Models/Azure.cs
index f202a22..ba0a205 100644
--- a/APSIM.POStats.Portal/Models/Azure.cs
+++ b/APSIM.POStats.Portal/Models/Azure.cs
@@ -9,7 +9,7 @@ namespace APSIM.POStats.Portal.Models
     /// <summary>
     /// Class for managing Azure Batch resources.
     /// </summary>
-    public class AzureBatchManager
+    public static class AzureBatchManager
     {
 
         /// <summary>
@@ -27,10 +27,9 @@ public static async Task CloseBatchPoolAsync(string poolName)
                 if (_batchClient == null)
                     throw new InvalidOperationException("Unable to get a Azure batch client while closing a pool.");
 
-                var pools = await _batchClient.PoolOperations.ListPools().ToListAsync();
-                CloudPool pool = pools.FirstOrDefault(p => p.Id == poolName);
-
+                CloudPool pool = _batchClient.PoolOperations.GetPoolAsync(poolName).Result ?? throw new InvalidOperationException($"Unable to find pool {poolName} while closing a pool.");
                 await _batchClient.PoolOperations.DeletePoolAsync(pool.Id);
+                Console.WriteLine($"Successfully deleted pool {poolName}.");
             }
             catch (BatchException be)
             {
@@ -64,9 +63,9 @@ private static BatchClient GetBatchClient()
                 throw new Exception("Cannot find variable AZURE_BATCH_ACCOUNT_KEY");
 
             return BatchClient.Open(new BatchSharedKeyCredentials(
-                Environment.GetEnvironmentVariable("AZURE_BATCH_ACCOUNT_URL"),
-                Environment.GetEnvironmentVariable("AZURE_BATCH_ACCOUNT_NAME"),
-                Environment.GetEnvironmentVariable("AZURE_BATCH_ACCOUNT_KEY")));
+                accountUrl,
+                accountName,
+                primaryAccessKey));
         }
     }
 }
\ No newline at end of file
diff --git a/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj b/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj
index efc24b4..ae2e38e 100644
--- a/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj
+++ b/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj
@@ -17,6 +17,7 @@
     <PackageReference Include="NUnit3TestAdapter" Version="5.0.0" />
     <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.18" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="16.3.1" />
   </ItemGroup>
 
   <ItemGroup>
@@ -25,6 +26,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\APSIM.POStats.Shared\APSIM.POStats.Shared.csproj" />
+    <ProjectReference Include="..\APSIM.POStats.Portal\APSIM.POStats.Portal.csproj" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/APSIM.POStats.Tests/AzureTests.cs b/APSIM.POStats.Tests/AzureTests.cs
new file mode 100644
index 0000000..9f8d5ea
--- /dev/null
+++ b/APSIM.POStats.Tests/AzureTests.cs
@@ -0,0 +1,32 @@
+using APSIM.POStats.Portal.Models;
+
+
+namespace APSIM.POStats.Tests;
+
+[TestFixture]
+public class AzureTests
+{
+    // This can be used for testing the CloseBatchPoolAsync method.
+    // Make sure you have a pool called "testpool" in your Azure Batch account before
+    // [Test]
+    // public async Task TestCloseBatchPoolAsync()
+    // {
+    //     using (File.OpenRead(".env"))
+    //     if (File.Exists(".env"))
+    //     {
+    //         var lines = File.ReadAllLines(".env");
+    //         foreach (var line in lines)
+    //         {
+    //             var parts = line.Split('=', 2);
+    //             if (parts.Length == 2)
+    //             {
+    //                 Environment.SetEnvironmentVariable(parts[0], parts[1]);
+    //             }
+    //         }
+    //     }
+    //     string poolName = "testpool";
+    //     await AzureBatchManager.CloseBatchPoolAsync(poolName);
+    // }
+
+}
+    
\ No newline at end of file

From 013411c82d52bf1935e74680e03b76c2ebc5bce2 Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Mon, 15 Sep 2025 12:34:05 +1000
Subject: [PATCH 25/48] Improve error messages regarding Azure batch pool value

---
 APSIM.POStats.Portal/Controllers/Api.cs        | 2 ++
 APSIM.POStats.Portal/Models/Azure.cs           | 8 +++++---
 APSIM.POStats.Tests/APSIM.POStats.Tests.csproj | 5 +++++
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index e61a8e5..c12c1ee 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -119,6 +119,8 @@ public async Task<IActionResult> Close(int pullrequestnumber, string commitid)
 
                     VariableComparison.Status status = PullRequestFunctions.GetStatus(pullRequest);
                     GitHub.SetStatus(pullrequestnumber, commitid, status);
+                    if (string.IsNullOrEmpty(pullRequest.Pool))
+                        throw new Exception("No pool associated with this pull request. Pool is required to close the Azure Batch pool.");
                     await AzureBatchManager.CloseBatchPoolAsync(pullRequest.Pool);
                 }
                 catch (Exception ex)
diff --git a/APSIM.POStats.Portal/Models/Azure.cs b/APSIM.POStats.Portal/Models/Azure.cs
index ba0a205..001dfa7 100644
--- a/APSIM.POStats.Portal/Models/Azure.cs
+++ b/APSIM.POStats.Portal/Models/Azure.cs
@@ -27,18 +27,20 @@ public static async Task CloseBatchPoolAsync(string poolName)
                 if (_batchClient == null)
                     throw new InvalidOperationException("Unable to get a Azure batch client while closing a pool.");
 
-                CloudPool pool = _batchClient.PoolOperations.GetPoolAsync(poolName).Result ?? throw new InvalidOperationException($"Unable to find pool {poolName} while closing a pool.");
+                CloudPool pool = _batchClient.PoolOperations.ListPools().Where(p => p.Id == poolName).FirstOrDefault();
+                if (pool == null)
+                    throw new InvalidOperationException($"Unable to find pool {poolName} while closing a pool.");
                 await _batchClient.PoolOperations.DeletePoolAsync(pool.Id);
                 Console.WriteLine($"Successfully deleted pool {poolName}.");
             }
             catch (BatchException be)
             {
-                Console.WriteLine(be.ToString());
+                Console.WriteLine($"Azure Batch pool with pool id {poolName} failed: " + be.ToString());
                 throw;
             }
             catch (Exception e)
             {
-                Console.WriteLine(e.ToString());
+                Console.WriteLine($"Azure Batch pool with pool id {poolName} failed: " + e.ToString());
                 throw;
             }
         }
diff --git a/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj b/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj
index ae2e38e..54314ec 100644
--- a/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj
+++ b/APSIM.POStats.Tests/APSIM.POStats.Tests.csproj
@@ -33,4 +33,9 @@
     <EmbeddedResource Include="Test.apsimx" />
   </ItemGroup>
 
+  <!-- Include the .env file for testing -->
+  <!-- <ItemGroup>
+    <Content Include=".env" CopyToOutputDirectory="PreserveNewest" />
+  </ItemGroup> -->
+
 </Project>

From c75852058262620d8cd8d07e30b37923c0b5c570 Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Mon, 15 Sep 2025 14:44:48 +1000
Subject: [PATCH 26/48] Fix Open PR endpoint and related methods to include
 pool property

---
 APSIM.POStats.Portal/Controllers/Api.cs | 9 +++++++--
 APSIM.POStats.Shared/StatsDbContext.cs  | 4 +++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index c12c1ee..a6a5d5c 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -61,10 +61,13 @@ public Api(StatsDbContext stats)
 
         /// <summary>Invoked by collector to open a pull request.</summary>
         /// <param name="pullrequestnumber">The number of the pull request to open.</param>
+        /// <param name="commitid">The commit id of the pull request.</param>
+        /// <param name="count">The number of expected validation tasks to be run.</param>
         /// <param name="author">The author of the pull request.</param>
+        /// <param name="pool">The name of the Azure Batch pool to be used for this pull request.</param>
         /// <returns></returns>
         [HttpGet("open")]
-        public IActionResult Open(int pullrequestnumber, string commitid, int count, string author)
+        public IActionResult Open(int pullrequestnumber, string commitid, int count, string author, string pool)
         {
             Console.WriteLine($"\"{author}\" opening PR \"{pullrequestnumber}\" with commit \"{commitid}\" and count \"{count}\"");
             if (pullrequestnumber == 0)
@@ -73,11 +76,13 @@ public IActionResult Open(int pullrequestnumber, string commitid, int count, str
                 return BadRequest("You must supply a commitid");
             if (string.IsNullOrEmpty(author))
                 return BadRequest("You must supply an author");
+            if (string.IsNullOrEmpty(pool))
+                return BadRequest("You must supply a pool name");
             try
             {
                 GitHub.SetStatus(pullrequestnumber, commitid, VariableComparison.Status.Running, $"Running {count} Validation Tasks");
 
-                statsDb.OpenPullRequest(pullrequestnumber, commitid, author, count);
+                statsDb.OpenPullRequest(pullrequestnumber, commitid, author, count, pool);
 
                 // Create a timer to check how many files have been returned
                 PullRequestTimer finishTimer = new PullRequestTimer { Interval = 10000, PullRequest = pullrequestnumber, Commit = commitid };
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 5e0aa4f..4c0abd9 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -34,7 +34,7 @@ public StatsDbContext(DbContextOptions<StatsDbContext> options)
         /// </remarks>
         /// <param name="pullRequestNumber">The pull request number.</param>
         /// <param name="author">The author of the pull request</param>
-        public void OpenPullRequest(int pullRequestNumber, string commitNumber, string author, int count)
+        public void OpenPullRequest(int pullRequestNumber, string commitNumber, string author, int count, string pool)
         {
             // Try and locate the pull request. If it doesn't exist, create a new pull request instance.
             // If it does exist, delete the old data.
@@ -62,6 +62,8 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
             pr.Status ??= new();
             pr.Status.Clear();
 
+            pr.Pool = pool;
+
             SaveChanges();
         }
 

From 48730be32f2081f5c119f7922341d917e644aa63 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Wed, 17 Sep 2025 09:47:30 +1000
Subject: [PATCH 27/48] Merging wheat back together

---
 APSIM.POStats.Shared/PullRequestFunctions.cs | 25 +++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/APSIM.POStats.Shared/PullRequestFunctions.cs b/APSIM.POStats.Shared/PullRequestFunctions.cs
index 68bdbf9..19d2675 100644
--- a/APSIM.POStats.Shared/PullRequestFunctions.cs
+++ b/APSIM.POStats.Shared/PullRequestFunctions.cs
@@ -43,11 +43,30 @@ public static VariableComparison.Status GetStatus(PullRequestDetails pullRequest
         public static List<ApsimFileComparison> GetFileComparisons(PullRequestDetails pullRequest)
         {
             var files = new List<ApsimFileComparison>();
-            foreach (var currentFile in pullRequest.Files)
+            foreach (ApsimFile currentFile in pullRequest.Files)
             {
-                var acceptedFile = pullRequest.AcceptedPullRequest?.Files.Find(f => f.Name == currentFile.Name);
-                files.Add(new ApsimFileComparison(currentFile, acceptedFile));
+                if (!currentFile.Name.Contains("Wheat-"))
+                {
+                    ApsimFile acceptedFile = pullRequest.AcceptedPullRequest?.Files.Find(f => f.Name == currentFile.Name);
+                    files.Add(new ApsimFileComparison(currentFile, acceptedFile));
+                }
+            }
+
+            //Merge Wheat back together
+            ApsimFile wheatFile = new ApsimFile();
+            wheatFile.Name = "Wheat";
+            foreach (ApsimFile currentFile in pullRequest.Files)
+            {
+                if (currentFile.Name.Contains("Wheat-"))
+                {
+                    wheatFile.Id = currentFile.Id;
+                    wheatFile.PullRequestId = currentFile.PullRequestId;
+                    wheatFile.PullRequest = currentFile.PullRequest;
+                    wheatFile.Tables.AddRange(currentFile.Tables);
+                }
             }
+            ApsimFile acceptedWheat = pullRequest.AcceptedPullRequest?.Files.Find(f => f.Name == wheatFile.Name);
+            files.Add(new ApsimFileComparison(wheatFile, acceptedWheat));
 
             // Add in files that are in the accepted PR but not in the current PR.
             if (pullRequest.AcceptedPullRequest != null)

From 63a89ad5019c0a42d621a1f1a4893548f899de79 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Wed, 17 Sep 2025 11:44:01 +1000
Subject: [PATCH 28/48] Initialise Tables

---
 APSIM.POStats.Shared/PullRequestFunctions.cs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/APSIM.POStats.Shared/PullRequestFunctions.cs b/APSIM.POStats.Shared/PullRequestFunctions.cs
index 19d2675..5028dd1 100644
--- a/APSIM.POStats.Shared/PullRequestFunctions.cs
+++ b/APSIM.POStats.Shared/PullRequestFunctions.cs
@@ -55,6 +55,7 @@ public static List<ApsimFileComparison> GetFileComparisons(PullRequestDetails pu
             //Merge Wheat back together
             ApsimFile wheatFile = new ApsimFile();
             wheatFile.Name = "Wheat";
+            wheatFile.Tables = new List<Table>();
             foreach (ApsimFile currentFile in pullRequest.Files)
             {
                 if (currentFile.Name.Contains("Wheat-"))

From 6c22d49100d1898ee30da5b779c8d3bc2c06fdbc Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Wed, 17 Sep 2025 13:16:44 +1000
Subject: [PATCH 29/48] Merge Wheat better

---
 APSIM.POStats.Shared/Models/ApsimFile.cs     | 31 ++++++++++
 APSIM.POStats.Shared/Models/Table.cs         | 65 ++++++++++++++------
 APSIM.POStats.Shared/PullRequestFunctions.cs |  6 +-
 3 files changed, 79 insertions(+), 23 deletions(-)

diff --git a/APSIM.POStats.Shared/Models/ApsimFile.cs b/APSIM.POStats.Shared/Models/ApsimFile.cs
index 298273c..bca7ec1 100644
--- a/APSIM.POStats.Shared/Models/ApsimFile.cs
+++ b/APSIM.POStats.Shared/Models/ApsimFile.cs
@@ -13,5 +13,36 @@ public class ApsimFile
         public int PullRequestId { get; set; }
         [JsonIgnore]
         public virtual PullRequestDetails PullRequest { get; set; }
+
+        public static ApsimFile Merge(ApsimFile fileA, ApsimFile fileB)
+        {
+            ApsimFile newFile = new ApsimFile();
+            newFile.Id = fileA.Id;
+            newFile.Name = fileA.Name;
+            newFile.PullRequestId = fileA.PullRequestId;
+            newFile.PullRequest = fileA.PullRequest;
+
+            fileA.Tables = new List<Table>();
+            foreach (Table a in fileA.Tables)
+            {
+                newFile.Tables.Add(a);
+            }
+
+            foreach (Table b in fileB.Tables)
+            {
+                bool found = false;
+                foreach (Table a in newFile.Tables)
+                {
+                    if (a.Name == b.Name)
+                    {
+                        a.Variables = Table.MergeVariables(a.Variables, b.Variables);
+                    }
+                }
+                if (!found)
+                    newFile.Tables.Add(b);
+            }
+
+            return newFile;
+        }
     }
 }
\ No newline at end of file
diff --git a/APSIM.POStats.Shared/Models/Table.cs b/APSIM.POStats.Shared/Models/Table.cs
index 2a2df21..05168f9 100644
--- a/APSIM.POStats.Shared/Models/Table.cs
+++ b/APSIM.POStats.Shared/Models/Table.cs
@@ -1,18 +1,47 @@
-using System.Collections.Generic;
-using System.Text.Json.Serialization;
-
-namespace APSIM.POStats.Shared.Models
-{
-    public class Table
-    {
-        public int Id { get; set; }
-        public string Name { get; set; }
-        public virtual List<Variable> Variables { get; set; }
-
-        [JsonIgnore]
-        public int ApsimFileId { get; set; }
-
-        [JsonIgnore]
-        public virtual ApsimFile ApsimFile { get; set; }
-    }
-}
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace APSIM.POStats.Shared.Models
+{
+    public class Table
+    {
+        public int Id { get; set; }
+        public string Name { get; set; }
+        public virtual List<Variable> Variables { get; set; }
+
+        [JsonIgnore]
+        public int ApsimFileId { get; set; }
+
+        [JsonIgnore]
+        public virtual ApsimFile ApsimFile { get; set; }
+
+        public static List<Variable> MergeVariables(List<Variable> variablesA, List<Variable> variablesB)
+        {
+            List<Variable> newList = new List<Variable>();
+
+            foreach (Variable a in variablesA)
+            {
+                newList.Add(a);
+            }
+
+            foreach (Variable b in variablesB)
+            {
+                bool found = false;
+                foreach (Variable a in newList)
+                {
+                    if (a.Name == b.Name)
+                    {
+                        a.Data.AddRange(b.Data);
+                    }
+                }
+                if (!found)
+                    newList.Add(b);
+            }
+
+            foreach (Variable variable in newList)
+                VariableFunctions.EnsureStatsAreCalculated(variable, forceRecalculate: true);
+
+            return newList;
+        }
+    }
+}
diff --git a/APSIM.POStats.Shared/PullRequestFunctions.cs b/APSIM.POStats.Shared/PullRequestFunctions.cs
index 5028dd1..8356b4c 100644
--- a/APSIM.POStats.Shared/PullRequestFunctions.cs
+++ b/APSIM.POStats.Shared/PullRequestFunctions.cs
@@ -55,15 +55,11 @@ public static List<ApsimFileComparison> GetFileComparisons(PullRequestDetails pu
             //Merge Wheat back together
             ApsimFile wheatFile = new ApsimFile();
             wheatFile.Name = "Wheat";
-            wheatFile.Tables = new List<Table>();
             foreach (ApsimFile currentFile in pullRequest.Files)
             {
                 if (currentFile.Name.Contains("Wheat-"))
                 {
-                    wheatFile.Id = currentFile.Id;
-                    wheatFile.PullRequestId = currentFile.PullRequestId;
-                    wheatFile.PullRequest = currentFile.PullRequest;
-                    wheatFile.Tables.AddRange(currentFile.Tables);
+                    wheatFile = ApsimFile.Merge(wheatFile, currentFile);
                 }
             }
             ApsimFile acceptedWheat = pullRequest.AcceptedPullRequest?.Files.Find(f => f.Name == wheatFile.Name);

From deff61c5092b6b59b8f5000dac293be7a50ae598 Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Wed, 17 Sep 2025 13:40:19 +1000
Subject: [PATCH 30/48] create commit sha specific images to enable rollback

---
 build-and-push-to-dockerhub.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/build-and-push-to-dockerhub.sh b/build-and-push-to-dockerhub.sh
index 726db09..18f1e1f 100644
--- a/build-and-push-to-dockerhub.sh
+++ b/build-and-push-to-dockerhub.sh
@@ -10,8 +10,12 @@ echo ""
 # build the image first
 ./build.sh
 
+simple-sha=$(git rev-parse --short HEAD)
+
 # push the images to Docker Hub
 docker push apsiminitiative/postats2-portal:latest
+docker push apsiminitiative/postats2-portal:$simple-sha
 docker push apsiminitiative/postats2-collector:latest
+docker push apsiminitiative/postats2-collector:$simple-sha
 
 echo "Done."
\ No newline at end of file

From cca1b2e21e88aabafcacbc15244b79b5390eb0ab Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Wed, 17 Sep 2025 14:29:22 +1000
Subject: [PATCH 31/48] fix script so that it actually creates the images with
 correct tags

---
 build-and-push-to-dockerhub.sh | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/build-and-push-to-dockerhub.sh b/build-and-push-to-dockerhub.sh
index 18f1e1f..34cc960 100644
--- a/build-and-push-to-dockerhub.sh
+++ b/build-and-push-to-dockerhub.sh
@@ -10,12 +10,20 @@ echo ""
 # build the image first
 ./build.sh
 
-simple-sha=$(git rev-parse --short HEAD)
+simplesha=$(git rev-parse --short HEAD)
 
-# push the images to Docker Hub
+# push the portal image
 docker push apsiminitiative/postats2-portal:latest
-docker push apsiminitiative/postats2-portal:$simple-sha
+
+# also tag with the git sha
+docker tag apsiminitiative/postats2-portal:latest apsiminitiative/postats2-portal:$simplesha
+docker push apsiminitiative/postats2-portal:$simplesha
+
+# push the collector image
 docker push apsiminitiative/postats2-collector:latest
-docker push apsiminitiative/postats2-collector:$simple-sha
+
+# also tag with the git sha
+docker tag apsiminitiative/postats2-collector:latest apsiminitiative/postats2-collector:$simplesha
+docker push apsiminitiative/postats2-collector:$simplesha
 
 echo "Done."
\ No newline at end of file

From f29895cd68b0f566c4c1a18d3df33a5017d98234 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Wed, 17 Sep 2025 14:44:46 +1000
Subject: [PATCH 32/48] Fix typo in merge code

---
 APSIM.POStats.Shared/Models/ApsimFile.cs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/APSIM.POStats.Shared/Models/ApsimFile.cs b/APSIM.POStats.Shared/Models/ApsimFile.cs
index bca7ec1..bb37189 100644
--- a/APSIM.POStats.Shared/Models/ApsimFile.cs
+++ b/APSIM.POStats.Shared/Models/ApsimFile.cs
@@ -22,7 +22,7 @@ public static ApsimFile Merge(ApsimFile fileA, ApsimFile fileB)
             newFile.PullRequestId = fileA.PullRequestId;
             newFile.PullRequest = fileA.PullRequest;
 
-            fileA.Tables = new List<Table>();
+            newFile.Tables = new List<Table>();
             foreach (Table a in fileA.Tables)
             {
                 newFile.Tables.Add(a);

From 5aae3563a0cbd9babb5639a9994a2e6ab43d2083 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Wed, 17 Sep 2025 15:03:40 +1000
Subject: [PATCH 33/48] Initialise merged file

---
 APSIM.POStats.Shared/PullRequestFunctions.cs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/APSIM.POStats.Shared/PullRequestFunctions.cs b/APSIM.POStats.Shared/PullRequestFunctions.cs
index 8356b4c..1ff4b9e 100644
--- a/APSIM.POStats.Shared/PullRequestFunctions.cs
+++ b/APSIM.POStats.Shared/PullRequestFunctions.cs
@@ -55,10 +55,14 @@ public static List<ApsimFileComparison> GetFileComparisons(PullRequestDetails pu
             //Merge Wheat back together
             ApsimFile wheatFile = new ApsimFile();
             wheatFile.Name = "Wheat";
+            wheatFile.Tables = new List<Table>();
             foreach (ApsimFile currentFile in pullRequest.Files)
             {
                 if (currentFile.Name.Contains("Wheat-"))
                 {
+                    wheatFile.Id = currentFile.Id;
+                    wheatFile.PullRequestId = currentFile.PullRequestId;
+                    wheatFile.PullRequest = currentFile.PullRequest;
                     wheatFile = ApsimFile.Merge(wheatFile, currentFile);
                 }
             }

From f5156ac40719dbe455ba3e1533ef114ff8a7a424 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Thu, 18 Sep 2025 09:30:37 +1000
Subject: [PATCH 34/48] Rewrite merging code to work faster

---
 APSIM.POStats.Shared/Models/ApsimFile.cs     |  1 +
 APSIM.POStats.Shared/Models/Table.cs         |  4 +-
 APSIM.POStats.Shared/PullRequestFunctions.cs | 48 +++++++++++++++++++-
 3 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/APSIM.POStats.Shared/Models/ApsimFile.cs b/APSIM.POStats.Shared/Models/ApsimFile.cs
index bb37189..ba8e6dc 100644
--- a/APSIM.POStats.Shared/Models/ApsimFile.cs
+++ b/APSIM.POStats.Shared/Models/ApsimFile.cs
@@ -36,6 +36,7 @@ public static ApsimFile Merge(ApsimFile fileA, ApsimFile fileB)
                     if (a.Name == b.Name)
                     {
                         a.Variables = Table.MergeVariables(a.Variables, b.Variables);
+                        found = true;
                     }
                 }
                 if (!found)
diff --git a/APSIM.POStats.Shared/Models/Table.cs b/APSIM.POStats.Shared/Models/Table.cs
index 05168f9..98db369 100644
--- a/APSIM.POStats.Shared/Models/Table.cs
+++ b/APSIM.POStats.Shared/Models/Table.cs
@@ -32,15 +32,13 @@ public static List<Variable> MergeVariables(List<Variable> variablesA, List<Vari
                     if (a.Name == b.Name)
                     {
                         a.Data.AddRange(b.Data);
+                        found = true;
                     }
                 }
                 if (!found)
                     newList.Add(b);
             }
 
-            foreach (Variable variable in newList)
-                VariableFunctions.EnsureStatsAreCalculated(variable, forceRecalculate: true);
-
             return newList;
         }
     }
diff --git a/APSIM.POStats.Shared/PullRequestFunctions.cs b/APSIM.POStats.Shared/PullRequestFunctions.cs
index 1ff4b9e..9e70c4e 100644
--- a/APSIM.POStats.Shared/PullRequestFunctions.cs
+++ b/APSIM.POStats.Shared/PullRequestFunctions.cs
@@ -56,6 +56,7 @@ public static List<ApsimFileComparison> GetFileComparisons(PullRequestDetails pu
             ApsimFile wheatFile = new ApsimFile();
             wheatFile.Name = "Wheat";
             wheatFile.Tables = new List<Table>();
+
             foreach (ApsimFile currentFile in pullRequest.Files)
             {
                 if (currentFile.Name.Contains("Wheat-"))
@@ -63,9 +64,54 @@ public static List<ApsimFileComparison> GetFileComparisons(PullRequestDetails pu
                     wheatFile.Id = currentFile.Id;
                     wheatFile.PullRequestId = currentFile.PullRequestId;
                     wheatFile.PullRequest = currentFile.PullRequest;
-                    wheatFile = ApsimFile.Merge(wheatFile, currentFile);
+
+                    foreach (Table table in currentFile.Tables)
+                    {
+                        Table wheatTable = null;
+                        foreach (Table t in wheatFile.Tables)
+                            if (t.Name == table.Name)
+                                wheatTable = t;
+                        if (wheatTable == null)
+                        {
+                            wheatTable = new Table();
+                            wheatTable.Id = table.Id;
+                            wheatTable.Name = table.Name;
+                            wheatTable.Variables = new List<Variable>();
+                            wheatTable.ApsimFileId = table.ApsimFileId;
+                            wheatTable.ApsimFile = wheatFile;
+                            wheatFile.Tables.Add(wheatTable);
+                        }
+
+                        foreach (Variable variable in table.Variables)
+                        {
+                            Variable wheatVar = null;
+                            foreach (Variable v in wheatTable.Variables)
+                                if (v.Name == variable.Name)
+                                    wheatVar = v;
+                            if (wheatVar == null)
+                            {
+                                wheatVar = new Variable();
+                                wheatVar.Id = variable.Id;
+                                wheatVar.Name = variable.Name;
+                                wheatVar.N = variable.N;
+                                wheatVar.RMSE = variable.RMSE;
+                                wheatVar.NSE = variable.NSE;
+                                wheatVar.RSR = variable.RSR;
+                                wheatVar.TableId = wheatTable.Id;
+                                wheatVar.Table = wheatTable;
+                                wheatVar.Data = new List<VariableData>();
+                                wheatTable.Variables.Add(wheatVar);
+                            }
+                            wheatVar.Data.AddRange(variable.Data);
+                        }
+                    }
                 }
             }
+
+            foreach (Table table in wheatFile.Tables)
+                foreach (Variable variable in table.Variables)
+                    VariableFunctions.EnsureStatsAreCalculated(variable, forceRecalculate: true);
+
             ApsimFile acceptedWheat = pullRequest.AcceptedPullRequest?.Files.Find(f => f.Name == wheatFile.Name);
             files.Add(new ApsimFileComparison(wheatFile, acceptedWheat));
 

From bd631607550700e4ae7ca07b538a9edd84000476 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Tue, 23 Sep 2025 11:28:04 +1000
Subject: [PATCH 35/48] working on merging code, in testing

---
 APSIM.POStats.Portal/Controllers/Api.cs      | 33 ++++---
 APSIM.POStats.Shared/Models/ApsimFile.cs     | 32 -------
 APSIM.POStats.Shared/Models/Table.cs         | 27 ------
 APSIM.POStats.Shared/PullRequestFunctions.cs | 73 +--------------
 APSIM.POStats.Shared/StatsDbContext.cs       | 98 +++++++++++++++++++-
 APSIM.POStats.Shared/WebUtilities.cs         |  2 +-
 APSIM.POStats.Tests/StatsDbTests.cs          |  6 +-
 7 files changed, 125 insertions(+), 146 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index a6a5d5c..aa06182 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -32,24 +32,31 @@ private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
             Console.WriteLine($"Tick");
             PullRequestTimer timer = source as PullRequestTimer;
 
-            Task<string> response = WebUtilities.GetAsync($"{url}api/count?pullrequestnumber={timer.PullRequest}&commitid={timer.Commit}");
-            response.Wait();
-
-            string result = response.Result.ToString();
-            if (!string.IsNullOrEmpty(result))
+            try
             {
-                int count = int.Parse(result);
+                Task<string> response = WebUtilities.GetAsync($"{url}api/count?pullrequestnumber={timer.PullRequest}&commitid={timer.Commit}");
+                response.Wait();
+                
+                string result = response.Result.ToString();
+                if (!string.IsNullOrEmpty(result))
+                {
+                    int count = int.Parse(result);
 
-                double minutes = (DateTime.Now - timer.StartTime).TotalMinutes;
+                    double minutes = (DateTime.Now - timer.StartTime).TotalMinutes;
 
-                Console.WriteLine($"{timer.PullRequest} ({timer.Commit}): count={count} and minutes={Math.Round(minutes, 1)}");
-                if (minutes >= FINAL_TIMEOUT)
-                {
-                    timer.Stop();
-                    response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequest}&commitId={timer.Commit}");
-                    response.Wait();
+                    Console.WriteLine($"{timer.PullRequest} ({timer.Commit}): count={count} and minutes={Math.Round(minutes, 1)}");
+                    if (minutes >= FINAL_TIMEOUT)
+                    {
+                        timer.Stop();
+                        response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequest}&commitId={timer.Commit}");
+                        response.Wait();
+                    }
                 }
             }
+            catch
+            {
+                timer.Stop();
+            }
         }
 
         /// <summary>Constructor.</summary>
diff --git a/APSIM.POStats.Shared/Models/ApsimFile.cs b/APSIM.POStats.Shared/Models/ApsimFile.cs
index ba8e6dc..298273c 100644
--- a/APSIM.POStats.Shared/Models/ApsimFile.cs
+++ b/APSIM.POStats.Shared/Models/ApsimFile.cs
@@ -13,37 +13,5 @@ public class ApsimFile
         public int PullRequestId { get; set; }
         [JsonIgnore]
         public virtual PullRequestDetails PullRequest { get; set; }
-
-        public static ApsimFile Merge(ApsimFile fileA, ApsimFile fileB)
-        {
-            ApsimFile newFile = new ApsimFile();
-            newFile.Id = fileA.Id;
-            newFile.Name = fileA.Name;
-            newFile.PullRequestId = fileA.PullRequestId;
-            newFile.PullRequest = fileA.PullRequest;
-
-            newFile.Tables = new List<Table>();
-            foreach (Table a in fileA.Tables)
-            {
-                newFile.Tables.Add(a);
-            }
-
-            foreach (Table b in fileB.Tables)
-            {
-                bool found = false;
-                foreach (Table a in newFile.Tables)
-                {
-                    if (a.Name == b.Name)
-                    {
-                        a.Variables = Table.MergeVariables(a.Variables, b.Variables);
-                        found = true;
-                    }
-                }
-                if (!found)
-                    newFile.Tables.Add(b);
-            }
-
-            return newFile;
-        }
     }
 }
\ No newline at end of file
diff --git a/APSIM.POStats.Shared/Models/Table.cs b/APSIM.POStats.Shared/Models/Table.cs
index 98db369..118d0a2 100644
--- a/APSIM.POStats.Shared/Models/Table.cs
+++ b/APSIM.POStats.Shared/Models/Table.cs
@@ -14,32 +14,5 @@ public class Table
 
         [JsonIgnore]
         public virtual ApsimFile ApsimFile { get; set; }
-
-        public static List<Variable> MergeVariables(List<Variable> variablesA, List<Variable> variablesB)
-        {
-            List<Variable> newList = new List<Variable>();
-
-            foreach (Variable a in variablesA)
-            {
-                newList.Add(a);
-            }
-
-            foreach (Variable b in variablesB)
-            {
-                bool found = false;
-                foreach (Variable a in newList)
-                {
-                    if (a.Name == b.Name)
-                    {
-                        a.Data.AddRange(b.Data);
-                        found = true;
-                    }
-                }
-                if (!found)
-                    newList.Add(b);
-            }
-
-            return newList;
-        }
     }
 }
diff --git a/APSIM.POStats.Shared/PullRequestFunctions.cs b/APSIM.POStats.Shared/PullRequestFunctions.cs
index 9e70c4e..0fbbb61 100644
--- a/APSIM.POStats.Shared/PullRequestFunctions.cs
+++ b/APSIM.POStats.Shared/PullRequestFunctions.cs
@@ -1,6 +1,7 @@
 using APSIM.POStats.Shared.Comparison;
 using APSIM.POStats.Shared.Models;
 using System.Collections.Generic;
+using System.Data.Common;
 using System.Linq;
 
 namespace APSIM.POStats.Shared
@@ -42,79 +43,15 @@ public static VariableComparison.Status GetStatus(PullRequestDetails pullRequest
         /// <param name="pullRequest">The pull request.</param>
         public static List<ApsimFileComparison> GetFileComparisons(PullRequestDetails pullRequest)
         {
-            var files = new List<ApsimFileComparison>();
-            foreach (ApsimFile currentFile in pullRequest.Files)
-            {
-                if (!currentFile.Name.Contains("Wheat-"))
-                {
-                    ApsimFile acceptedFile = pullRequest.AcceptedPullRequest?.Files.Find(f => f.Name == currentFile.Name);
-                    files.Add(new ApsimFileComparison(currentFile, acceptedFile));
-                }
-            }
-
-            //Merge Wheat back together
-            ApsimFile wheatFile = new ApsimFile();
-            wheatFile.Name = "Wheat";
-            wheatFile.Tables = new List<Table>();
+            StatsDbContext.MergeSplitFiles(pullRequest, "Wheat-", "Wheat");
 
+            var files = new List<ApsimFileComparison>();
             foreach (ApsimFile currentFile in pullRequest.Files)
             {
-                if (currentFile.Name.Contains("Wheat-"))
-                {
-                    wheatFile.Id = currentFile.Id;
-                    wheatFile.PullRequestId = currentFile.PullRequestId;
-                    wheatFile.PullRequest = currentFile.PullRequest;
-
-                    foreach (Table table in currentFile.Tables)
-                    {
-                        Table wheatTable = null;
-                        foreach (Table t in wheatFile.Tables)
-                            if (t.Name == table.Name)
-                                wheatTable = t;
-                        if (wheatTable == null)
-                        {
-                            wheatTable = new Table();
-                            wheatTable.Id = table.Id;
-                            wheatTable.Name = table.Name;
-                            wheatTable.Variables = new List<Variable>();
-                            wheatTable.ApsimFileId = table.ApsimFileId;
-                            wheatTable.ApsimFile = wheatFile;
-                            wheatFile.Tables.Add(wheatTable);
-                        }
-
-                        foreach (Variable variable in table.Variables)
-                        {
-                            Variable wheatVar = null;
-                            foreach (Variable v in wheatTable.Variables)
-                                if (v.Name == variable.Name)
-                                    wheatVar = v;
-                            if (wheatVar == null)
-                            {
-                                wheatVar = new Variable();
-                                wheatVar.Id = variable.Id;
-                                wheatVar.Name = variable.Name;
-                                wheatVar.N = variable.N;
-                                wheatVar.RMSE = variable.RMSE;
-                                wheatVar.NSE = variable.NSE;
-                                wheatVar.RSR = variable.RSR;
-                                wheatVar.TableId = wheatTable.Id;
-                                wheatVar.Table = wheatTable;
-                                wheatVar.Data = new List<VariableData>();
-                                wheatTable.Variables.Add(wheatVar);
-                            }
-                            wheatVar.Data.AddRange(variable.Data);
-                        }
-                    }
-                }
+                ApsimFile acceptedFile = pullRequest.AcceptedPullRequest?.Files.Find(f => f.Name == currentFile.Name);
+                files.Add(new ApsimFileComparison(currentFile, acceptedFile));
             }
 
-            foreach (Table table in wheatFile.Tables)
-                foreach (Variable variable in table.Variables)
-                    VariableFunctions.EnsureStatsAreCalculated(variable, forceRecalculate: true);
-
-            ApsimFile acceptedWheat = pullRequest.AcceptedPullRequest?.Files.Find(f => f.Name == wheatFile.Name);
-            files.Add(new ApsimFileComparison(wheatFile, acceptedWheat));
-
             // Add in files that are in the accepted PR but not in the current PR.
             if (pullRequest.AcceptedPullRequest != null)
             {
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 4c0abd9..24e2e74 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -1,9 +1,10 @@
 using APSIM.POStats.Shared.Models;
-using Microsoft.CodeAnalysis.CSharp.Syntax;
+using System.Collections.Generic;
 using Microsoft.EntityFrameworkCore;
 using System;
 using System.Linq;
 using System.Threading;
+using SQLitePCL;
 
 namespace APSIM.POStats.Shared
 {
@@ -75,7 +76,7 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
         /// </remarks>
         /// <param name="pullRequest">The pull request to copy the data from..</param>
         /// <returns>Reference to stored PullRequest</returns>
-        public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullRequest, int retryCount = 0)
+        public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullRequest)
         {
             var pr = new PullRequestDetails();
 
@@ -101,6 +102,91 @@ public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullReques
             return pr;
         }
 
+        /// <summary>Get a list of all files for a pull request.</summary>
+        /// <param name="pullRequest">The pull request.</param>
+        public static void MergeSplitFiles(PullRequestDetails pullRequest, string prefix, string newName)
+        {
+            //Merge back together any files that have been split
+            List<ApsimFile> wheatFiles = new List<ApsimFile>();
+
+            //Merge Wheat back together
+            ApsimFile wheatFile = new ApsimFile();
+            wheatFile.Name = newName;
+            wheatFile.Tables = new List<Table>();
+
+            foreach (ApsimFile currentFile in pullRequest.Files)
+            {
+                if (currentFile.Name.Contains(prefix))
+                {
+                    wheatFiles.Add(currentFile);
+
+                    foreach (Table table in currentFile.Tables)
+                    {
+                        Table wheatTable = null;
+                        foreach (Table t in wheatFile.Tables)
+                            if (t.Name == table.Name)
+                                wheatTable = t;
+
+                        if (wheatTable == null)
+                        {
+                            wheatTable = new Table();
+                            wheatTable.Name = table.Name;
+                            wheatTable.Variables = new List<Variable>();
+                            wheatFile.Tables.Add(wheatTable);
+                        }
+
+                        foreach (Variable variable in table.Variables)
+                        {
+                            Variable wheatVar = null;
+                            foreach (Variable v in wheatTable.Variables)
+                                if (v.Name == variable.Name)
+                                    wheatVar = v;
+                            if (wheatVar == null)
+                            {
+                                wheatVar = new Variable();
+                                wheatVar.Name = variable.Name;
+                                wheatVar.N = variable.N;
+                                wheatVar.RMSE = variable.RMSE;
+                                wheatVar.NSE = variable.NSE;
+                                wheatVar.RSR = variable.RSR;
+                                wheatVar.Data = new List<VariableData>();
+                                wheatTable.Variables.Add(wheatVar);
+                            }
+                            foreach (VariableData data in variable.Data)
+                            {
+                                VariableData wheatData = new VariableData();
+                                wheatData.Label = data.Label;
+                                wheatData.Predicted = data.Predicted;
+                                wheatData.Observed = data.Observed;
+                                wheatVar.Data.Add(wheatData);
+                            }
+                        }
+                    }
+                }
+            }
+
+            pullRequest.Files.Add(wheatFile);
+            //foreach (ApsimFile file in wheatFiles)
+            //{
+            //    pullRequest.Files.Remove(file);
+            //}
+            
+            // Calculate stats for each variable in each table in each file.
+            foreach (var file in pullRequest.Files)
+            {
+                Console.WriteLine(file.Name);
+                foreach (var table in file.Tables)
+                {
+                    foreach (var variable in table.Variables)
+                    {
+                        VariableFunctions.EnsureStatsAreCalculated(variable);
+                    }
+                }
+            }
+
+            //SaveChanges();
+        }
+
         /// <summary>
         /// Add file data to a pull request.
         /// </summary>
@@ -162,9 +248,17 @@ public PullRequestDetails ClosePullRequest(int pullRequestNumber)
 
             // Calculate stats for each variable in each table in each file.
             foreach (var file in pr.Files)
+            {
+                Console.WriteLine(file.Name);
                 foreach (var table in file.Tables)
+                {
                     foreach (var variable in table.Variables)
+                    {
                         VariableFunctions.EnsureStatsAreCalculated(variable);
+                    }
+                }
+            }
+
             SaveChanges();
             return pr;
         }
diff --git a/APSIM.POStats.Shared/WebUtilities.cs b/APSIM.POStats.Shared/WebUtilities.cs
index f26c52b..042ed9e 100644
--- a/APSIM.POStats.Shared/WebUtilities.cs
+++ b/APSIM.POStats.Shared/WebUtilities.cs
@@ -62,7 +62,7 @@ private static async Task<string> RequestAsync(RequestType type, string requestU
                     output += $"Response:\n{body}\n";
 
                     Console.WriteLine($"Error sending POST Request\n{output}");
-                    throw new Exception();
+                    throw new Exception($"Error sending POST Request\n{output}");
                 }
                 return body;
             }
diff --git a/APSIM.POStats.Tests/StatsDbTests.cs b/APSIM.POStats.Tests/StatsDbTests.cs
index 1c1db48..47af66b 100644
--- a/APSIM.POStats.Tests/StatsDbTests.cs
+++ b/APSIM.POStats.Tests/StatsDbTests.cs
@@ -14,7 +14,7 @@ public void TestOpenPullRequestThatAlreadyExists()
     {
         using var db = CreateInMemoryDB("db1");
 
-        db.OpenPullRequest(1234, "1", "author", 0);
+        db.OpenPullRequest(1234, "1", "author", 0, "");
 
         // Make sure the pr exists and has no data.
         var pr = db.PullRequests.First(pr => pr.PullRequest == 1234);
@@ -29,7 +29,7 @@ public void TestOpenPullRequestThatDoesntAlreadyExist()
     {
         using var db = CreateInMemoryDB("db2");
 
-        db.OpenPullRequest(5678, "1", "author", 0);
+        db.OpenPullRequest(5678, "1", "author", 0, "");
 
         // Make sure the pr exists and has no ydata.
         var pr = db.PullRequests.First(pr => pr.PullRequest == 5678);
@@ -44,7 +44,7 @@ public void TestAddDataToPullRequest()
     {
         using var db = CreateInMemoryDB("db3");
 
-        db.OpenPullRequest(1234, "1", "author", 0);
+        db.OpenPullRequest(1234, "1", "author", 0, "");
         PullRequestDetails prToAdd = new()
         {
             PullRequest = 1234,

From e5177e42bc9132f1e0b2cfd4471f5cad6f5448f4 Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Tue, 23 Sep 2025 14:48:55 +1000
Subject: [PATCH 36/48] add count total to the timer to enable closure of PR
 when all jobs returned

---
 APSIM.POStats.Portal/Controllers/Api.cs  | 4 ++--
 APSIM.POStats.Shared/PullRequestTimer.cs | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index aa06182..337bab8 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -45,7 +45,7 @@ private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
                     double minutes = (DateTime.Now - timer.StartTime).TotalMinutes;
 
                     Console.WriteLine($"{timer.PullRequest} ({timer.Commit}): count={count} and minutes={Math.Round(minutes, 1)}");
-                    if (minutes >= FINAL_TIMEOUT)
+                    if (minutes >= FINAL_TIMEOUT || count >= timer.CountTotal)
                     {
                         timer.Stop();
                         response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequest}&commitId={timer.Commit}");
@@ -92,7 +92,7 @@ public IActionResult Open(int pullrequestnumber, string commitid, int count, str
                 statsDb.OpenPullRequest(pullrequestnumber, commitid, author, count, pool);
 
                 // Create a timer to check how many files have been returned
-                PullRequestTimer finishTimer = new PullRequestTimer { Interval = 10000, PullRequest = pullrequestnumber, Commit = commitid };
+                PullRequestTimer finishTimer = new PullRequestTimer { Interval = 10000, PullRequest = pullrequestnumber, Commit = commitid, CountTotal = count };
                 finishTimer.Elapsed += OnCheckIfFinished;
                 finishTimer.AutoReset = true;
                 finishTimer.StartTime = DateTime.Now;
diff --git a/APSIM.POStats.Shared/PullRequestTimer.cs b/APSIM.POStats.Shared/PullRequestTimer.cs
index 413f864..54bcdcf 100644
--- a/APSIM.POStats.Shared/PullRequestTimer.cs
+++ b/APSIM.POStats.Shared/PullRequestTimer.cs
@@ -12,5 +12,8 @@ public class PullRequestTimer : System.Timers.Timer
 
         /// <summary>Start Time</summary>
         public DateTime StartTime;
+
+        /// <summary>Number of nodes that should return</summary>
+        public int CountTotal;
     }
 }
\ No newline at end of file

From 911132f9ced343e4d4e1f565aca4d623b44f72fe Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Thu, 25 Sep 2025 15:14:35 +1000
Subject: [PATCH 37/48] POStats2 fully functional

---
 APSIM.POStats.Portal/Controllers/Api.cs      |  11 +-
 APSIM.POStats.Portal/Pages/Index.cshtml      |  13 ++
 APSIM.POStats.Shared/PullRequestFunctions.cs |   2 -
 APSIM.POStats.Shared/StatsDbContext.cs       | 118 +++++++++----------
 4 files changed, 77 insertions(+), 67 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index 337bab8..6ad4e14 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -20,7 +20,7 @@ public class Api : ControllerBase
         private static object _lock = new();
 
         /// <summary>Final timeout in minutes. Controls how long to wait before finally closing a pull request.</summary>
-        private const double FINAL_TIMEOUT = 30.0;
+        private const double FINAL_TIMEOUT = 40.0;
 
         /// <summary>Event handler for finish timer.</summary>
         private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
@@ -29,7 +29,6 @@ private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
             if (string.IsNullOrEmpty(url))
                 throw new Exception($"Cannot find environment variable POSTATS_UPLOAD_URL");
 
-            Console.WriteLine($"Tick");
             PullRequestTimer timer = source as PullRequestTimer;
 
             try
@@ -44,7 +43,7 @@ private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
 
                     double minutes = (DateTime.Now - timer.StartTime).TotalMinutes;
 
-                    Console.WriteLine($"{timer.PullRequest} ({timer.Commit}): count={count} and minutes={Math.Round(minutes, 1)}");
+                    //Console.WriteLine($"{timer.PullRequest} ({timer.Commit}): count={count} and minutes={Math.Round(minutes, 1)}");
                     if (minutes >= FINAL_TIMEOUT || count >= timer.CountTotal)
                     {
                         timer.Stop();
@@ -159,7 +158,7 @@ public async Task<IActionResult> Close(int pullrequestnumber, string commitid)
         [HttpGet("count")]
         public IActionResult Count(int pullrequestnumber, string commitid)
         {
-            Console.WriteLine($"api/count");
+            //Console.WriteLine($"api/count");
             
             if (pullrequestnumber == 0)
                 return BadRequest("You must supply a pull request number");
@@ -172,7 +171,7 @@ public IActionResult Count(int pullrequestnumber, string commitid)
                 try
                 {
                     var count = statsDb.GetNumberOfCompletesInPullRequest(pullrequestnumber, commitid);
-                    Console.WriteLine($"{count}");
+                    //Console.WriteLine($"{count}");
                     return Ok(count);
                 }
                 catch (Exception ex)
@@ -197,7 +196,7 @@ public IActionResult Count(int pullrequestnumber, string commitid)
         [RequestSizeLimit(100_000_000)]
         public IActionResult PostAsync([FromBody]PullRequestDetails pullrequest)
         {
-            Console.WriteLine($"api/adddata called");
+            //Console.WriteLine($"api/adddata called");
 
             if (pullrequest == null)
                 return BadRequest("You must supply a pull request");
diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml b/APSIM.POStats.Portal/Pages/Index.cshtml
index 37e0a05..0d7b0dd 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml
@@ -83,6 +83,19 @@
             <div>
                 <label for="ReturnCount">Return Count:</label>
                 <input name="ReturnCount" value="@Model.PullRequest.Status.Count" readonly />
+                <label for="TotalCount"> of </label>
+                <input name="TotalCount" value="@Model.PullRequest.CountTotal" readonly />
+            </div>
+            <div>
+                <label for="Status">Status: </label>
+                @if (@Model.PullRequest.Status.Count != @Model.PullRequest.CountTotal)
+                {
+                    <label style="color:orange;font-weight:bold">Running</label>
+                }
+                else
+                {
+                    <label style="color:green;font-weight:bold">Completed</label>
+                }
             </div>
             <br />
             <br />
diff --git a/APSIM.POStats.Shared/PullRequestFunctions.cs b/APSIM.POStats.Shared/PullRequestFunctions.cs
index 0fbbb61..edc85d0 100644
--- a/APSIM.POStats.Shared/PullRequestFunctions.cs
+++ b/APSIM.POStats.Shared/PullRequestFunctions.cs
@@ -43,8 +43,6 @@ public static VariableComparison.Status GetStatus(PullRequestDetails pullRequest
         /// <param name="pullRequest">The pull request.</param>
         public static List<ApsimFileComparison> GetFileComparisons(PullRequestDetails pullRequest)
         {
-            StatsDbContext.MergeSplitFiles(pullRequest, "Wheat-", "Wheat");
-
             var files = new List<ApsimFileComparison>();
             foreach (ApsimFile currentFile in pullRequest.Files)
             {
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 24e2e74..206d869 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -106,85 +106,83 @@ public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullReques
         /// <param name="pullRequest">The pull request.</param>
         public static void MergeSplitFiles(PullRequestDetails pullRequest, string prefix, string newName)
         {
-            //Merge back together any files that have been split
-            List<ApsimFile> wheatFiles = new List<ApsimFile>();
-
-            //Merge Wheat back together
-            ApsimFile wheatFile = new ApsimFile();
-            wheatFile.Name = newName;
-            wheatFile.Tables = new List<Table>();
-
+            List<VariableData> copiedData = new List<VariableData>();
+            List<string> variableRef = new List<string>();
+            List<string> tableRef = new List<string>();
+
+            //This looks like a weird way to do this, however for DBContext to work
+            //We cannot incrementally add data to the structure, otherwise it breaks
+            //So we have work everything out first, then work backwards filling in our 
+            //lists all at once before moving to the next variable/table.
+
+            //We will need to re-write this structure in the future to all for better
+            //data sorting and lookup.
+            List<ApsimFile> splitFiles = new List<ApsimFile>();
             foreach (ApsimFile currentFile in pullRequest.Files)
             {
                 if (currentFile.Name.Contains(prefix))
                 {
-                    wheatFiles.Add(currentFile);
-
+                    splitFiles.Add(currentFile);
                     foreach (Table table in currentFile.Tables)
                     {
-                        Table wheatTable = null;
-                        foreach (Table t in wheatFile.Tables)
-                            if (t.Name == table.Name)
-                                wheatTable = t;
-
-                        if (wheatTable == null)
-                        {
-                            wheatTable = new Table();
-                            wheatTable.Name = table.Name;
-                            wheatTable.Variables = new List<Variable>();
-                            wheatFile.Tables.Add(wheatTable);
-                        }
-
+                        if (!tableRef.Contains(table.Name))
+                            tableRef.Add(table.Name);
                         foreach (Variable variable in table.Variables)
                         {
-                            Variable wheatVar = null;
-                            foreach (Variable v in wheatTable.Variables)
-                                if (v.Name == variable.Name)
-                                    wheatVar = v;
-                            if (wheatVar == null)
-                            {
-                                wheatVar = new Variable();
-                                wheatVar.Name = variable.Name;
-                                wheatVar.N = variable.N;
-                                wheatVar.RMSE = variable.RMSE;
-                                wheatVar.NSE = variable.NSE;
-                                wheatVar.RSR = variable.RSR;
-                                wheatVar.Data = new List<VariableData>();
-                                wheatTable.Variables.Add(wheatVar);
-                            }
+                            if (!variableRef.Contains(variable.Name))
+                                variableRef.Add(variable.Name);
+
                             foreach (VariableData data in variable.Data)
-                            {
-                                VariableData wheatData = new VariableData();
-                                wheatData.Label = data.Label;
-                                wheatData.Predicted = data.Predicted;
-                                wheatData.Observed = data.Observed;
-                                wheatVar.Data.Add(wheatData);
-                            }
+                                copiedData.Add(data);
                         }
                     }
                 }
             }
 
-            pullRequest.Files.Add(wheatFile);
-            //foreach (ApsimFile file in wheatFiles)
-            //{
-            //    pullRequest.Files.Remove(file);
-            //}
-            
-            // Calculate stats for each variable in each table in each file.
-            foreach (var file in pullRequest.Files)
+            List<Table> tables = new List<Table>();
+            foreach (string table in tableRef.Distinct())
             {
-                Console.WriteLine(file.Name);
-                foreach (var table in file.Tables)
+                List<Variable> variables = new List<Variable>();
+                foreach (string variable in variableRef.Distinct())
                 {
-                    foreach (var variable in table.Variables)
+                    List<VariableData> datas = new List<VariableData>();
+                    foreach (VariableData data in copiedData)
                     {
-                        VariableFunctions.EnsureStatsAreCalculated(variable);
+                        if (data.Variable.Table.Name == table && data.Variable.Name == variable)
+                        {
+                            VariableData newData = new VariableData();
+                            newData.Label = data.Label;
+                            newData.Predicted = data.Predicted;
+                            newData.Observed = data.Observed;
+                            datas.Add(newData);
+                        }
+                    }
+                    if (datas.Count() > 0)
+                    {
+                        Variable newVariable = new Variable();
+                        newVariable.Name = variable;
+                        newVariable.Data = datas;
+                        variables.Add(newVariable);
                     }
                 }
+                if (variables.Count() > 0)
+                {
+                    Table newTable = new Table();
+                    newTable.Name = table;
+                    newTable.Variables = variables;
+                    tables.Add(newTable);
+                }
             }
 
-            //SaveChanges();
+            //Merge Wheat back together
+            ApsimFile combinedFile = new ApsimFile();
+            combinedFile.Name = newName;
+            combinedFile.Tables = tables;
+
+            pullRequest.Files.Add(combinedFile);
+            foreach (ApsimFile file in splitFiles)
+                pullRequest.Files.Remove(file);
+
         }
 
         /// <summary>
@@ -246,10 +244,12 @@ public PullRequestDetails ClosePullRequest(int pullRequestNumber)
             // Assign the current accepted pull request.
             pr.AcceptedPullRequest = GetMostRecentAcceptedPullRequest();
 
+            StatsDbContext.MergeSplitFiles(pr, "Wheat-", "Wheat");
+            StatsDbContext.MergeSplitFiles(pr, "FAR-", "FAR");
+
             // Calculate stats for each variable in each table in each file.
             foreach (var file in pr.Files)
             {
-                Console.WriteLine(file.Name);
                 foreach (var table in file.Tables)
                 {
                     foreach (var variable in table.Variables)

From f24ce9280875833d4c1949a35798416788fcdecb Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Mon, 6 Oct 2025 17:20:27 +1000
Subject: [PATCH 38/48] Add more detail back to users about exceptions and
 running status

---
 APSIM.POStats.Portal/Controllers/Api.cs      |   5 +-
 APSIM.POStats.Portal/Pages/Index.cshtml      | 192 ++++++++++---------
 APSIM.POStats.Shared/GitHub/GitHub.cs        |  10 +-
 APSIM.POStats.Shared/PullRequestFunctions.cs |  12 +-
 4 files changed, 121 insertions(+), 98 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index 6ad4e14..33935bb 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -49,6 +49,7 @@ private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
                         timer.Stop();
                         response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequest}&commitId={timer.Commit}");
                         response.Wait();
+                        GitHub.SetStatus(timer.PullRequest, timer.Commit, VariableComparison.Status.Missing, "Timeout Error");
                     }
                 }
             }
@@ -86,10 +87,10 @@ public IActionResult Open(int pullrequestnumber, string commitid, int count, str
                 return BadRequest("You must supply a pool name");
             try
             {
-                GitHub.SetStatus(pullrequestnumber, commitid, VariableComparison.Status.Running, $"Running {count} Validation Tasks");
-
                 statsDb.OpenPullRequest(pullrequestnumber, commitid, author, count, pool);
 
+                GitHub.SetStatus(pullrequestnumber, commitid, VariableComparison.Status.Running, $"Running {count} Validation Tasks");
+
                 // Create a timer to check how many files have been returned
                 PullRequestTimer finishTimer = new PullRequestTimer { Interval = 10000, PullRequest = pullrequestnumber, Commit = commitid, CountTotal = count };
                 finishTimer.Elapsed += OnCheckIfFinished;
diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml b/APSIM.POStats.Portal/Pages/Index.cshtml
index 0d7b0dd..4c29672 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml
@@ -88,30 +88,27 @@
             </div>
             <div>
                 <label for="Status">Status: </label>
-                @if (@Model.PullRequest.Status.Count != @Model.PullRequest.CountTotal)
+                @if (PullRequestFunctions.HasExceptionInLogs(@Model.PullRequest))
+                {
+                    <label style="color:red;font-weight:bold">Exception Thrown</label>
+                }
+                else if (@Model.PullRequest.Status.Count != @Model.PullRequest.CountTotal)
                 {
                     <label style="color:orange;font-weight:bold">Running</label>
                 }
-                else
+                else if (@Model.PullRequest.AcceptedPullRequest == null)
                 {
-                    <label style="color:green;font-weight:bold">Completed</label>
+                    <label style="color:purple;font-weight:bold">Calculating Stats</label>
                 }
-            </div>
-            <br />
-            <br />
-            <div>
-                <input type="hidden" name="stats-value" value="@Model.OnlyShowChangedStats.ToString()" readonly />
-                @if (Model.OnlyShowChangedStats)
+                else if (PullRequestFunctions.GetStatus(@Model.PullRequest) != VariableComparison.Status.Same)
                 {
-                    <button name="stats-button" type="submit" value="True">Show all stats</button>
+                    <label style="color:red;font-weight:bold">Completed with Stats Changes</label>
                 }
                 else
                 {
-                    <button name="stats-button" type="submit" value="True">Show changed stats</button>
+                    <label style="color:green;font-weight:bold">Completed</label>
                 }
             </div>
-            <br />
-            <br />
             <div>
                 <input type="hidden" name="logs-value" value="@Model.ShowLogs.ToString()" readonly />
                 @if (Model.ShowLogs)
@@ -123,100 +120,119 @@
                     <button name="logs-button" type="submit" value="True">Show Logs</button>
                 }
             </div>
-            <br />
-            <br />
-            <div>
-                <label for="Filter">Filter By:</label>
-                <input name="filter-value" value="@Model.Filter" />
-                <button name="filter-button" type="submit" value="True">Enter</button>
-            </div>
+            @if (@Model.PullRequest.AcceptedPullRequest != null)
+            {
+                <div>
+                    <input type="hidden" name="stats-value" value="@Model.OnlyShowChangedStats.ToString()" readonly />
+                    @if (Model.OnlyShowChangedStats)
+                    {
+                        <button name="stats-button" type="submit" value="True">Show all stats</button>
+                    }
+                    else
+                    {
+                        <button name="stats-button" type="submit" value="True">Show changed stats</button>
+                    }
+                </div>
+                <div>
+                    <label for="Filter">Filter By:</label>
+                    <input name="filter-value" value="@Model.Filter" />
+                    <button name="filter-button" type="submit" value="True">Enter</button>
+                </div>
+            }
         </form>
 
-        <table class="stats-table">
-            @{
-                List<ApsimFileComparison> files = PullRequestFunctions.GetFileComparisons(Model.PullRequest);
-                if (Model.OnlyShowChangedStats)
-                    PullRequestFunctions.RemoveSame(files);
-            }
-            @foreach (var file in files)
-            {
-                @if (IndexModel.FilterVariables(files, Model.Filter, file.Name, "", ""))
+        @if (@Model.PullRequest.AcceptedPullRequest != null)
+        {
+            <table class="stats-table">
+                @{
+                    List<ApsimFileComparison> files = PullRequestFunctions.GetFileComparisons(Model.PullRequest);
+                    if (Model.OnlyShowChangedStats)
+                        PullRequestFunctions.RemoveSame(files);
+                }
+                @foreach (var file in files)
                 {
-                    <tr></tr>
-                    <tr>
-                        <td class="stats-table-heading" colspan="18">
-                            <b>@Html.Raw(IndexModel.EmitFileName(file))</b>
-                        </td>
-                    </tr>
-                    @foreach (var table in file.Tables)
+                    @if (IndexModel.FilterVariables(files, Model.Filter, file.Name, "", ""))
                     {
-                        @if (IndexModel.FilterVariables(files, Model.Filter, file.Name, table.Name, ""))
+                        <tr></tr>
+                        <tr>
+                            <td class="stats-table-heading" colspan="18">
+                                <b>@Html.Raw(IndexModel.EmitFileName(file))</b>
+                            </td>
+                        </tr>
+                        @foreach (var table in file.Tables)
                         {
-                            <tr>
-                                <td class=".stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
-                                <td class="stats-table-heading" colspan="2"><i>@Html.Raw(IndexModel.EmitTableName(@table))</i></td>
-                                <td class="stats-heading">&nbsp;&nbsp;</td>
-                                <td class="stats-heading" colspan="3">n</td>
-                                <td class="stats-heading">&nbsp;&nbsp;</td>
-                                <td class="stats-heading" colspan="3">RMSE</td>
-                                <td class="stats-heading">&nbsp;&nbsp;</td>
-                                <td class="stats-heading" colspan="3">NSE</td>
-                                <td class="stats-heading">&nbsp;&nbsp;</td>
-                                <td class="stats-heading" colspan="3">RSR</td>
-                            </tr>
-
-                            @foreach (var variable in table.VariableComparisons)
+                            @if (IndexModel.FilterVariables(files, Model.Filter, file.Name, table.Name, ""))
                             {
-                                @if (IndexModel.FilterVariables(files, Model.Filter, file.Name, table.Name, variable.Name))
+                                <tr>
+                                    <td class=".stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
+                                    <td class="stats-table-heading" colspan="2"><i>@Html.Raw(IndexModel.EmitTableName(@table))</i></td>
+                                    <td class="stats-heading">&nbsp;&nbsp;</td>
+                                    <td class="stats-heading" colspan="3">n</td>
+                                    <td class="stats-heading">&nbsp;&nbsp;</td>
+                                    <td class="stats-heading" colspan="3">RMSE</td>
+                                    <td class="stats-heading">&nbsp;&nbsp;</td>
+                                    <td class="stats-heading" colspan="3">NSE</td>
+                                    <td class="stats-heading">&nbsp;&nbsp;</td>
+                                    <td class="stats-heading" colspan="3">RSR</td>
+                                </tr>
+
+                                @foreach (var variable in table.VariableComparisons)
                                 {
-                                    <tr>
-                                        <td class="stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
-                                        <td class="stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
-                                        <td class="stats-variablename"><a asp-page="/Chart" asp-route-id="@variable.Id">@Html.Raw(IndexModel.EmitVariableName(@variable))</a></td>
-                                        <td class="stats-cell">&nbsp;&nbsp;</td>
+                                    @if (IndexModel.FilterVariables(files, Model.Filter, file.Name, table.Name, variable.Name))
+                                    {
+                                        <tr>
+                                            <td class="stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
+                                            <td class="stats-variablename">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</td>
+                                            <td class="stats-variablename"><a asp-page="/Chart" asp-route-id="@variable.Id">@Html.Raw(IndexModel.EmitVariableName(@variable))</a></td>
+                                            <td class="stats-cell">&nbsp;&nbsp;</td>
 
-                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentN, isAccepted: false))</td>
-                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedN, isAccepted: true))</td>
-                                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.NStatus))</td>
-                                        <td class="stats-cell">&nbsp;&nbsp;</td>
+                                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentN, isAccepted: false))</td>
+                                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedN, isAccepted: true))</td>
+                                            <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.NStatus))</td>
+                                            <td class="stats-cell">&nbsp;&nbsp;</td>
 
-                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(variable.CurrentRMSE, 0, 6, isAccepted: false))</td>
-                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(variable.AcceptedRMSE, 0, 6, isAccepted: true))</td>
-                                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.RMSEStatus))</td>
-                                        <td class="stats-cell">&nbsp;&nbsp;</td>
+                                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(variable.CurrentRMSE, 0, 6, isAccepted: false))</td>
+                                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(variable.AcceptedRMSE, 0, 6, isAccepted: true))</td>
+                                            <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.RMSEStatus))</td>
+                                            <td class="stats-cell">&nbsp;&nbsp;</td>
 
-                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentNSE, 3, 0, isAccepted: false))</td>
-                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedNSE, 3, 0, isAccepted: true))</td>
-                                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.NSEStatus))@VariableFunctions.NSERating(variable.CurrentNSE)</td>
-                                        <td class="stats-cell">&nbsp;&nbsp;</td>
+                                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentNSE, 3, 0, isAccepted: false))</td>
+                                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedNSE, 3, 0, isAccepted: true))</td>
+                                            <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.NSEStatus))@VariableFunctions.NSERating(variable.CurrentNSE)</td>
+                                            <td class="stats-cell">&nbsp;&nbsp;</td>
 
-                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentRSR, 3, 0, isAccepted: false))</td>
-                                        <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedRSR, 3, 0, isAccepted: true))</td>
-                                        <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.RSRStatus))@VariableFunctions.RSRRating(variable.CurrentRSR)</td>
-                                    </tr>
+                                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.CurrentRSR, 3, 0, isAccepted: false))</td>
+                                            <td class="stats-cell">&nbsp;@Html.Raw(IndexModel.EmitNumber(@variable.AcceptedRSR, 3, 0, isAccepted: true))</td>
+                                            <td class="stats-tick">@Html.Raw(IndexModel.EmitTickCross(variable.RSRStatus))@VariableFunctions.RSRRating(variable.CurrentRSR)</td>
+                                        </tr>
+                                    }
                                 }
                             }
                         }
                     }
                 }
-            }
-        </table>
-        <div class="output-logs">
-            Logs:
-            @if (Model.ShowLogs) 
-            {
-                @foreach (string log in Model.PullRequest.Outputs)
+            </table>
+        }
+        else
+        {
+            <div>Validation in progress</div>
+        }
+            <div class="output-logs">
+                Logs:
+                @if (Model.ShowLogs)
                 {
-                    @foreach (string line in log.Split("\n"))
+                    @foreach (string log in Model.PullRequest.Outputs)
                     {
-                        @Html.Encode(line)<br />
+                        @foreach (string line in log.Split("\n"))
+                        {
+                            @Html.Encode(line)<br />
+                        }
                     }
                 }
-            }
-            else
-            {
-                @Html.Raw("Hidden")
-            }
-        </div>
+                else
+                {
+                    @Html.Raw("Hidden")
+                }
+            </div>
     </div>
 </div>
diff --git a/APSIM.POStats.Shared/GitHub/GitHub.cs b/APSIM.POStats.Shared/GitHub/GitHub.cs
index 18bb73a..259fe19 100644
--- a/APSIM.POStats.Shared/GitHub/GitHub.cs
+++ b/APSIM.POStats.Shared/GitHub/GitHub.cs
@@ -58,20 +58,19 @@ public static void SetStatus(int pullRequestNumber, string commitId, VariableCom
             if (pullRequestTask.State.ToLower() != "open")
                 throw new Exception($"Cannot set the status of a Pull Request that is not open (id: {pullRequestNumber})");
 
-            //if (!pullRequestTask.StatusURL.Contains(commitId))
-            //    throw new Exception($"Cannot set the status of a Pull Request as the commit id is not the latest (PR: {pullRequestNumber}, Commit:{commitId})");
+            if (!pullRequestTask.StatusURL.Contains(commitId))
+                throw new Exception($"Cannot set the status of a Pull Request as the commit id is not the latest (PR: {pullRequestNumber}, Commit:{commitId})");
 
             //check the status of POStats for this PR
             string state = "failure";
            
-
             if (status == VariableComparison.Status.Same)
                 state = "success";
 
             if (status == VariableComparison.Status.Running)
                 state = "pending";
 
-            string stateFormatted = "In Development: " + status.ToString();
+            string stateFormatted = status.ToString();
             if (!String.IsNullOrEmpty(message))
                 stateFormatted = message;
 
@@ -81,9 +80,6 @@ public static void SetStatus(int pullRequestNumber, string commitId, VariableCom
 
             Console.WriteLine($"Github Status Update: {state} {urlStr} {stateFormatted} {"APSIM.POStats2"}");
 
-            //Always success for now
-            state = "success";
-
             //Status POST body details
             GitHubStatusDetails body = new GitHubStatusDetails(state, urlStr, stateFormatted, "APSIM.POStats2");
 
diff --git a/APSIM.POStats.Shared/PullRequestFunctions.cs b/APSIM.POStats.Shared/PullRequestFunctions.cs
index edc85d0..2d45f6b 100644
--- a/APSIM.POStats.Shared/PullRequestFunctions.cs
+++ b/APSIM.POStats.Shared/PullRequestFunctions.cs
@@ -21,7 +21,7 @@ public static VariableComparison.Status GetStatus(PullRequestDetails pullRequest
             {
                 if (file.Status != ApsimFileComparison.StatusType.NoChange)
                     return VariableComparison.Status.Different;
-                
+
                 foreach (var table in file.Tables)
                 {
                     if (table.Status != ApsimFileComparison.StatusType.NoChange)
@@ -89,5 +89,15 @@ public static void UpdateStats(PullRequestDetails pullRequest)
                     foreach (var variable in table.Variables)
                         VariableFunctions.EnsureStatsAreCalculated(variable, forceRecalculate: true);
         }
+
+        /// <summary>Update the stats in the specified pull request.</summary>
+        /// <param name="pullRequest"></param>
+        public static bool HasExceptionInLogs(PullRequestDetails pullRequest)
+        {
+            foreach (string log in pullRequest.Outputs)
+                if (log.Contains("Exception"))
+                    return true;
+            return false;
+        }
     }
 }

From b91283026eeb3869f87f4f2f11a76d07e0d8a93a Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Mon, 6 Oct 2025 17:29:30 +1000
Subject: [PATCH 39/48] Return exception message to github when one is thrown

---
 APSIM.POStats.Portal/Controllers/Api.cs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index 33935bb..9306e27 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -129,10 +129,19 @@ public async Task<IActionResult> Close(int pullrequestnumber, string commitid)
                     // Send pass/fail to gitHub
                     var pullRequest = statsDb.ClosePullRequest(pullrequestnumber);
 
-                    VariableComparison.Status status = PullRequestFunctions.GetStatus(pullRequest);
-                    GitHub.SetStatus(pullrequestnumber, commitid, status);
+                    if (PullRequestFunctions.HasExceptionInLogs(pullRequest))
+                    {
+                        GitHub.SetStatus(pullrequestnumber, commitid, VariableComparison.Status.Missing, "Exception Thrown");
+                    }
+                    else
+                    {
+                        VariableComparison.Status status = PullRequestFunctions.GetStatus(pullRequest);
+                        GitHub.SetStatus(pullrequestnumber, commitid, status);
+                    }
+
                     if (string.IsNullOrEmpty(pullRequest.Pool))
                         throw new Exception("No pool associated with this pull request. Pool is required to close the Azure Batch pool.");
+                        
                     await AzureBatchManager.CloseBatchPoolAsync(pullRequest.Pool);
                 }
                 catch (Exception ex)

From e252d139f66d4dae08bd20dc0a34be0e3ac3bc20 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Wed, 8 Oct 2025 09:15:29 +1000
Subject: [PATCH 40/48] Add check for timeout

---
 APSIM.POStats.Portal/Controllers/Api.cs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index 9306e27..ec2b2b1 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -43,13 +43,14 @@ private static void OnCheckIfFinished(Object source, ElapsedEventArgs e)
 
                     double minutes = (DateTime.Now - timer.StartTime).TotalMinutes;
 
-                    //Console.WriteLine($"{timer.PullRequest} ({timer.Commit}): count={count} and minutes={Math.Round(minutes, 1)}");
                     if (minutes >= FINAL_TIMEOUT || count >= timer.CountTotal)
                     {
                         timer.Stop();
                         response = WebUtilities.GetAsync($"{url}api/close?pullRequestNumber={timer.PullRequest}&commitId={timer.Commit}");
                         response.Wait();
-                        GitHub.SetStatus(timer.PullRequest, timer.Commit, VariableComparison.Status.Missing, "Timeout Error");
+
+                        if (count < timer.CountTotal)
+                            GitHub.SetStatus(timer.PullRequest, timer.Commit, VariableComparison.Status.Missing, "Timeout Error");
                     }
                 }
             }

From 88339528dd1a8a01a498d81d38cea1e214aeb56d Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Thu, 9 Oct 2025 08:00:44 +1000
Subject: [PATCH 41/48] Stopwatches

---
 APSIM.POStats.Portal/Controllers/Api.cs | 24 +++++-------------------
 APSIM.POStats.Portal/Pages/Index.cshtml |  4 ++--
 APSIM.POStats.Shared/StatsDbContext.cs  | 21 +++++++++++++++------
 3 files changed, 22 insertions(+), 27 deletions(-)

diff --git a/APSIM.POStats.Portal/Controllers/Api.cs b/APSIM.POStats.Portal/Controllers/Api.cs
index ec2b2b1..b2a560c 100644
--- a/APSIM.POStats.Portal/Controllers/Api.cs
+++ b/APSIM.POStats.Portal/Controllers/Api.cs
@@ -93,7 +93,7 @@ public IActionResult Open(int pullrequestnumber, string commitid, int count, str
                 GitHub.SetStatus(pullrequestnumber, commitid, VariableComparison.Status.Running, $"Running {count} Validation Tasks");
 
                 // Create a timer to check how many files have been returned
-                PullRequestTimer finishTimer = new PullRequestTimer { Interval = 10000, PullRequest = pullrequestnumber, Commit = commitid, CountTotal = count };
+                PullRequestTimer finishTimer = new PullRequestTimer { Interval = 60000, PullRequest = pullrequestnumber, Commit = commitid, CountTotal = count };
                 finishTimer.Elapsed += OnCheckIfFinished;
                 finishTimer.AutoReset = true;
                 finishTimer.StartTime = DateTime.Now;
@@ -182,7 +182,6 @@ public IActionResult Count(int pullrequestnumber, string commitid)
                 try
                 {
                     var count = statsDb.GetNumberOfCompletesInPullRequest(pullrequestnumber, commitid);
-                    //Console.WriteLine($"{count}");
                     return Ok(count);
                 }
                 catch (Exception ex)
@@ -207,31 +206,18 @@ public IActionResult Count(int pullrequestnumber, string commitid)
         [RequestSizeLimit(100_000_000)]
         public IActionResult PostAsync([FromBody]PullRequestDetails pullrequest)
         {
-            //Console.WriteLine($"api/adddata called");
-
             if (pullrequest == null)
                 return BadRequest("You must supply a pull request");
 
             lock (_lock)
             {
-                if (statsDb.PullRequestWithCommitExists(pullrequest.PullRequest, pullrequest.Commit))
+                try
                 {
-                    try
-                    {
-                        PullRequestDetails pr = statsDb.AddDataToPullRequest(pullrequest);
-                    }
-                    catch (Exception ex)
-                    {
-                        return BadRequest(ex.ToString());
-                    }
+                    PullRequestDetails pr = statsDb.AddDataToPullRequest(pullrequest);
                 }
-                else
+                catch (Exception ex)
                 {
-                    PullRequestDetails pr = statsDb.GetPullRequest(pullrequest.PullRequest);
-                    if (pr == null)
-                        return BadRequest($"A PR with {pullrequest.PullRequest} does not exist in the database");
-                    else
-                        return BadRequest($"A PR with {pullrequest.PullRequest} does exist, but has commit number {pr.Commit}, and you submitted a commit of {pullrequest.Commit}");
+                    return BadRequest(ex.ToString());
                 }
             }
             return Ok();
diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml b/APSIM.POStats.Portal/Pages/Index.cshtml
index 4c29672..f823bb2 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml
@@ -82,7 +82,7 @@
             </div>
             <div>
                 <label for="ReturnCount">Return Count:</label>
-                <input name="ReturnCount" value="@Model.PullRequest.Status.Count" readonly />
+                <input name="ReturnCount" value="@Model.PullRequest.Outputs.Count" readonly />
                 <label for="TotalCount"> of </label>
                 <input name="TotalCount" value="@Model.PullRequest.CountTotal" readonly />
             </div>
@@ -92,7 +92,7 @@
                 {
                     <label style="color:red;font-weight:bold">Exception Thrown</label>
                 }
-                else if (@Model.PullRequest.Status.Count != @Model.PullRequest.CountTotal)
+                else if (@Model.PullRequest.Outputs.Count != @Model.PullRequest.CountTotal)
                 {
                     <label style="color:orange;font-weight:bold">Running</label>
                 }
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 206d869..bdbe21b 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -4,7 +4,7 @@
 using System;
 using System.Linq;
 using System.Threading;
-using SQLitePCL;
+using System.Diagnostics;
 
 namespace APSIM.POStats.Shared
 {
@@ -78,6 +78,10 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
         /// <returns>Reference to stored PullRequest</returns>
         public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullRequest)
         {
+            string file = fromPullRequest.Files.FirstOrDefault().Name;
+            string number = fromPullRequest.PullRequest.ToString();
+            Stopwatch stopwatch = Stopwatch.StartNew();
+
             var pr = new PullRequestDetails();
 
             // Find the pull request. Should always exist if OpenPullRequest has been called.
@@ -85,20 +89,25 @@ public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullReques
             if (pr == null)
                 throw new Exception($"Cannot find POStats pull request number: {fromPullRequest.PullRequest}");
 
-            foreach (ApsimFile file in fromPullRequest.Files)
-                Console.WriteLine($"File \"{file.Name}\" added to PR {fromPullRequest.PullRequest}");
+            stopwatch.Stop();
+            Console.WriteLine($"\"{file}\" found PR {number} in {stopwatch.ElapsedMilliseconds} ms");
 
+            stopwatch = Stopwatch.StartNew();
             if (fromPullRequest.Files != null)
                 pr.Files.AddRange(fromPullRequest.Files);
 
             if (fromPullRequest.Outputs != null)
                 pr.Outputs.AddRange(fromPullRequest.Outputs);
 
-            if (fromPullRequest.Status != null)
-                pr.Status.AddRange(fromPullRequest.Status);
+            stopwatch.Stop();
+            Console.WriteLine($"\"{file}\" copied to PR {number} in {stopwatch.ElapsedMilliseconds} ms");
 
+            stopwatch = Stopwatch.StartNew();
             SaveChanges();
 
+            stopwatch.Stop();
+            Console.WriteLine($"\"{file}\" written to PR {number} in {stopwatch.ElapsedMilliseconds} ms");
+
             return pr;
         }
 
@@ -202,7 +211,7 @@ public int GetNumberOfCompletesInPullRequest(int pullrequestnumber, string commi
             if (pr == null)
                 throw new Exception($"Cannot find POStats pull request number: {pullrequestnumber}");
 
-            return pr.Status.Count;
+            return pr.Outputs.Count;
         }
 
         public bool SaveChangesMultipleTries(int retries = 0)

From af0556e0d3101737945bf5b72d1e3b0551b0a8b5 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Thu, 9 Oct 2025 09:49:37 +1000
Subject: [PATCH 42/48] Add indexing, removed stopwatches

---
 .../Models/PullRequestDetails.cs              |  2 ++
 APSIM.POStats.Shared/StatsDbContext.cs        | 31 ++++++++++++-------
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/APSIM.POStats.Shared/Models/PullRequestDetails.cs b/APSIM.POStats.Shared/Models/PullRequestDetails.cs
index 1566f59..4fb8f6a 100644
--- a/APSIM.POStats.Shared/Models/PullRequestDetails.cs
+++ b/APSIM.POStats.Shared/Models/PullRequestDetails.cs
@@ -1,8 +1,10 @@
 using System;
 using System.Collections.Generic;
+using Microsoft.EntityFrameworkCore;
 
 namespace APSIM.POStats.Shared.Models
 {
+    [Index(nameof(PullRequest), IsUnique = true)]
     public class PullRequestDetails
     {
         public int Id { get; set; }
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index bdbe21b..725ee39 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -80,33 +80,38 @@ public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullReques
         {
             string file = fromPullRequest.Files.FirstOrDefault().Name;
             string number = fromPullRequest.PullRequest.ToString();
-            Stopwatch stopwatch = Stopwatch.StartNew();
+            //Stopwatch stopwatch = Stopwatch.StartNew();
 
             var pr = new PullRequestDetails();
 
             // Find the pull request. Should always exist if OpenPullRequest has been called.
-            pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == fromPullRequest.PullRequest && pr.Commit == fromPullRequest.Commit);
+            pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == fromPullRequest.PullRequest);
             if (pr == null)
                 throw new Exception($"Cannot find POStats pull request number: {fromPullRequest.PullRequest}");
+            if (pr.Commit != fromPullRequest.Commit)
+                throw new Exception($"PR {fromPullRequest.PullRequest}:{fromPullRequest.Commit} is not the latest commit {pr.Commit}, cannot save stats.");
 
-            stopwatch.Stop();
-            Console.WriteLine($"\"{file}\" found PR {number} in {stopwatch.ElapsedMilliseconds} ms");
+            //stopwatch.Stop();
+            //Console.WriteLine($"\"{file}\" found PR {number} in {stopwatch.ElapsedMilliseconds} ms");
 
-            stopwatch = Stopwatch.StartNew();
+            //stopwatch = Stopwatch.StartNew();
             if (fromPullRequest.Files != null)
                 pr.Files.AddRange(fromPullRequest.Files);
 
             if (fromPullRequest.Outputs != null)
                 pr.Outputs.AddRange(fromPullRequest.Outputs);
 
-            stopwatch.Stop();
-            Console.WriteLine($"\"{file}\" copied to PR {number} in {stopwatch.ElapsedMilliseconds} ms");
+            if (fromPullRequest.Status != null)
+                pr.Status.AddRange(fromPullRequest.Status);
 
-            stopwatch = Stopwatch.StartNew();
+            //stopwatch.Stop();
+            //Console.WriteLine($"\"{file}\" copied to PR {number} in {stopwatch.ElapsedMilliseconds} ms");
+
+            //stopwatch = Stopwatch.StartNew();
             SaveChanges();
 
-            stopwatch.Stop();
-            Console.WriteLine($"\"{file}\" written to PR {number} in {stopwatch.ElapsedMilliseconds} ms");
+            //stopwatch.Stop();
+            //Console.WriteLine($"\"{file}\" written to PR {number} in {stopwatch.ElapsedMilliseconds} ms");
 
             return pr;
         }
@@ -207,11 +212,13 @@ public int GetNumberOfCompletesInPullRequest(int pullrequestnumber, string commi
             var pr = new PullRequestDetails();
 
             // Find the pull request. Should always exist if OpenPullRequest has been called.
-            pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == pullrequestnumber && pr.Commit == commitid);
+            pr = PullRequests.FirstOrDefault(pr => pr.PullRequest == pullrequestnumber);
             if (pr == null)
                 throw new Exception($"Cannot find POStats pull request number: {pullrequestnumber}");
+            if (pr.Commit != pr.Commit)
+                throw new Exception($"PR {pullrequestnumber}:{commitid} is not the latest commit {pr.Commit}, cannot save stats.");
 
-            return pr.Outputs.Count;
+            return pr.Status.Count;
         }
 
         public bool SaveChangesMultipleTries(int retries = 0)

From 07f1244ddf0bc2bb02878c328fe15fca843b6142 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Tue, 28 Oct 2025 16:33:25 +1000
Subject: [PATCH 43/48] Add phenology and Eucalyptus to merge list

---
 APSIM.POStats.Shared/StatsDbContext.cs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 725ee39..a5fe677 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -262,6 +262,8 @@ public PullRequestDetails ClosePullRequest(int pullRequestNumber)
 
             StatsDbContext.MergeSplitFiles(pr, "Wheat-", "Wheat");
             StatsDbContext.MergeSplitFiles(pr, "FAR-", "FAR");
+            StatsDbContext.MergeSplitFiles(pr, "WheatPhenology-", "WheatPhenology");
+            StatsDbContext.MergeSplitFiles(pr, "Eucalyptus-", "Eucalyptus");
 
             // Calculate stats for each variable in each table in each file.
             foreach (var file in pr.Files)

From db2c8be059ed7cc271d253f19d466abf6afcf1eb Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Wed, 29 Oct 2025 09:30:17 +1000
Subject: [PATCH 44/48] Fix status count on index page

---
 APSIM.POStats.Portal/Pages/Index.cshtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml b/APSIM.POStats.Portal/Pages/Index.cshtml
index f823bb2..13aaa15 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml
@@ -82,7 +82,7 @@
             </div>
             <div>
                 <label for="ReturnCount">Return Count:</label>
-                <input name="ReturnCount" value="@Model.PullRequest.Outputs.Count" readonly />
+                <input name="ReturnCount" value="@Model.PullRequest.Status.Count" readonly />
                 <label for="TotalCount"> of </label>
                 <input name="TotalCount" value="@Model.PullRequest.CountTotal" readonly />
             </div>

From 63bd4963dd66c0c2dccf114b5c48f007900ea3f0 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Wed, 29 Oct 2025 11:36:36 +1000
Subject: [PATCH 45/48] Fixes to get it working again

---
 APSIM.POStats.Collector/Program.cs      | 1 +
 APSIM.POStats.Portal/Pages/Index.cshtml | 2 +-
 APSIM.POStats.Shared/StatsDbContext.cs  | 6 +++---
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/APSIM.POStats.Collector/Program.cs b/APSIM.POStats.Collector/Program.cs
index 731c04d..0b91931 100644
--- a/APSIM.POStats.Collector/Program.cs
+++ b/APSIM.POStats.Collector/Program.cs
@@ -91,6 +91,7 @@ static int Main(string[] args)
                     Stopwatch stopwatch = Stopwatch.StartNew();
                     UploadStats(pullRequest, url);
                     Console.WriteLine($"Elapsed time to send data to new web api: {stopwatch.Elapsed.TotalSeconds} seconds");
+                    stopwatch.Stop();
                 }
                 else if (command == "close")
                 {
diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml b/APSIM.POStats.Portal/Pages/Index.cshtml
index 13aaa15..4c29672 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml
@@ -92,7 +92,7 @@
                 {
                     <label style="color:red;font-weight:bold">Exception Thrown</label>
                 }
-                else if (@Model.PullRequest.Outputs.Count != @Model.PullRequest.CountTotal)
+                else if (@Model.PullRequest.Status.Count != @Model.PullRequest.CountTotal)
                 {
                     <label style="color:orange;font-weight:bold">Running</label>
                 }
diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index a5fe677..b8605b1 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -78,8 +78,8 @@ public void OpenPullRequest(int pullRequestNumber, string commitNumber, string a
         /// <returns>Reference to stored PullRequest</returns>
         public PullRequestDetails AddDataToPullRequest(PullRequestDetails fromPullRequest)
         {
-            string file = fromPullRequest.Files.FirstOrDefault().Name;
-            string number = fromPullRequest.PullRequest.ToString();
+            //string file = fromPullRequest.Files.FirstOrDefault().Name;
+            //string number = fromPullRequest.PullRequest.ToString();
             //Stopwatch stopwatch = Stopwatch.StartNew();
 
             var pr = new PullRequestDetails();
@@ -263,7 +263,7 @@ public PullRequestDetails ClosePullRequest(int pullRequestNumber)
             StatsDbContext.MergeSplitFiles(pr, "Wheat-", "Wheat");
             StatsDbContext.MergeSplitFiles(pr, "FAR-", "FAR");
             StatsDbContext.MergeSplitFiles(pr, "WheatPhenology-", "WheatPhenology");
-            StatsDbContext.MergeSplitFiles(pr, "Eucalyptus-", "Eucalyptus");
+            //StatsDbContext.MergeSplitFiles(pr, "Eucalyptus-", "Eucalyptus");
 
             // Calculate stats for each variable in each table in each file.
             foreach (var file in pr.Files)

From 0de1d7a49f2c8ef920bd1969fca5beeb683f2b7d Mon Sep 17 00:00:00 2001
From: Julian Rich <123442863+ric394@users.noreply.github.com>
Date: Mon, 3 Nov 2025 16:06:06 +1000
Subject: [PATCH 46/48] Fix ratings table

---
 APSIM.POStats.Portal/Pages/Index.cshtml | 11 +++++------
 APSIM.POStats.Shared/GitHub/GitHub.cs   |  4 ++--
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/APSIM.POStats.Portal/Pages/Index.cshtml b/APSIM.POStats.Portal/Pages/Index.cshtml
index 4c29672..30b2ab4 100644
--- a/APSIM.POStats.Portal/Pages/Index.cshtml
+++ b/APSIM.POStats.Portal/Pages/Index.cshtml
@@ -25,12 +25,11 @@
             </tr>
             <tr class="ratings-table-heading">
                 <td class="ratings-table-column">&nbsp;</td>
-                <td class="ratings-tab
-    using Microsoft.EntityFrameworkCore.Metadata.Internal;
-
-#nullable disable
-#line default
-        <tr class="ratings-table-row">
+                <td class="ratings-table-column">Rating</td>
+                <td class="ratings-table-column">RSR</td>
+                <td class="ratings-table-column">NSE</td>
+            </tr>
+            <tr class="ratings-table-row">
                 <td class="ratings-table-column">***</td>
                 <td class="ratings-table-column">Very Good</td>
                 <td class="ratings-table-column">0.00 ≤ RSR ≤ 0.50</td>
diff --git a/APSIM.POStats.Shared/GitHub/GitHub.cs b/APSIM.POStats.Shared/GitHub/GitHub.cs
index 259fe19..3fac93b 100644
--- a/APSIM.POStats.Shared/GitHub/GitHub.cs
+++ b/APSIM.POStats.Shared/GitHub/GitHub.cs
@@ -78,10 +78,10 @@ public static void SetStatus(int pullRequestNumber, string commitId, VariableCom
             string serverURL = Environment.GetEnvironmentVariable("POSTATS_UPLOAD_URL");
             string urlStr = $"{serverURL}{pullRequestNumber}";
 
-            Console.WriteLine($"Github Status Update: {state} {urlStr} {stateFormatted} {"APSIM.POStats2"}");
+            Console.WriteLine($"Github Status Update: {state} {urlStr} {stateFormatted} {"APSIM Acceptance Tests"}");
 
             //Status POST body details
-            GitHubStatusDetails body = new GitHubStatusDetails(state, urlStr, stateFormatted, "APSIM.POStats2");
+            GitHubStatusDetails body = new GitHubStatusDetails(state, urlStr, stateFormatted, "APSIM Acceptance Tests");
 
             //Send POST request
             Task<string> response = WebUtilities.PostAsync(pullRequestTask.StatusURL, body, token);

From e730accf25bd261a3a0cb508a3a2b97c6de4db04 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Mon, 5 Jan 2026 13:42:10 +1000
Subject: [PATCH 47/48] Add Eucalyptus splitter

---
 APSIM.POStats.Shared/StatsDbContext.cs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index b8605b1..8af51ce 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -263,7 +263,7 @@ public PullRequestDetails ClosePullRequest(int pullRequestNumber)
             StatsDbContext.MergeSplitFiles(pr, "Wheat-", "Wheat");
             StatsDbContext.MergeSplitFiles(pr, "FAR-", "FAR");
             StatsDbContext.MergeSplitFiles(pr, "WheatPhenology-", "WheatPhenology");
-            //StatsDbContext.MergeSplitFiles(pr, "Eucalyptus-", "Eucalyptus");
+            StatsDbContext.MergeSplitFiles(pr, "Eucalyptus-", "Eucalyptus");
 
             // Calculate stats for each variable in each table in each file.
             foreach (var file in pr.Files)

From 651282fe2a806f2d45a312d3edeb4809e1e50021 Mon Sep 17 00:00:00 2001
From: Andrew Paroz <par456@csiro.au>
Date: Mon, 12 Jan 2026 14:06:48 +1000
Subject: [PATCH 48/48] Fix broken unit test

---
 APSIM.POStats.Shared/StatsDbContext.cs | 5 +++--
 APSIM.POStats.Tests/StatsDbTests.cs    | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/APSIM.POStats.Shared/StatsDbContext.cs b/APSIM.POStats.Shared/StatsDbContext.cs
index 8af51ce..fb23c04 100644
--- a/APSIM.POStats.Shared/StatsDbContext.cs
+++ b/APSIM.POStats.Shared/StatsDbContext.cs
@@ -193,10 +193,11 @@ public static void MergeSplitFiles(PullRequestDetails pullRequest, string prefix
             combinedFile.Name = newName;
             combinedFile.Tables = tables;
 
-            pullRequest.Files.Add(combinedFile);
+            if (combinedFile.Tables.Count > 0)
+                pullRequest.Files.Add(combinedFile);
+            
             foreach (ApsimFile file in splitFiles)
                 pullRequest.Files.Remove(file);
-
         }
 
         /// <summary>
diff --git a/APSIM.POStats.Tests/StatsDbTests.cs b/APSIM.POStats.Tests/StatsDbTests.cs
index 47af66b..466e9ce 100644
--- a/APSIM.POStats.Tests/StatsDbTests.cs
+++ b/APSIM.POStats.Tests/StatsDbTests.cs
@@ -53,6 +53,7 @@ public void TestAddDataToPullRequest()
             {
                 new ApsimFile()
                 {
+                    Name = "Table",
                     Tables = new()
                     {
                         new Table()