Deploying AL-Go from main (4000fc21109200985749fbbb6b62d22d20a8adcb) to main (#74)

Deploying AL-Go from main (4000fc21109200985749fbbb6b62d22d20a8adcb) to
main

Co-authored-by: microsoft <[email protected]>

Author: bcbuild-github-agent

AL-Go-Helper.ps1

@@ -43,6 +43,23 @@ function Test-Shell {
     }
 }
 
+function Test-Deprecations {
+    Param(
+        [HashTable] $json,
+        [string] $settingsDescription
+    )
+
+    # cleanModePreprocessorSymbols is deprecated
+    if ($json.Keys -contains 'cleanModePreprocessorSymbols') {
+        OutputWarning -Message "cleanModePreprocessorSymbols in $settingsDescription is deprecated. See https://aka.ms/algodeprecations#cleanModePreprocessorSymbols"
+    }
+
+    # <workflowName>Schedule is deprecated
+    ($json.Keys | Where-Object {$_ -like '*Schedule' -and $_ -ne 'WorkflowSchedule'}) | ForEach-Object {
+        OutputWarning -Message "$_ in $settingsDescription is deprecated. See https://aka.ms/algodeprecations#_workflow_Schedule. This warning will become an error in the future."
+    }
+}
+
 function Test-SettingsJson {
     Param(
         [hashtable] $json,
@@ -51,6 +68,8 @@ function Test-SettingsJson {
         [string] $type
     )
 
+    Test-Deprecations -json $json -settingsDescription $settingsDescription
+
     Test-Shell -json $json -settingsDescription $settingsDescription -property 'shell'
     Test-Shell -json $json -settingsDescription $settingsDescription -property 'gitHubRunnerShell'
 
@@ -71,18 +90,17 @@ function Test-SettingsJson {
     if ($type -eq 'Workflow') {
         # Test for things that should / should not exist in a workflow settings file
     }
+    else {
+        # workflowSchedule and workflowConcurrency should only exist in workflow specific settings files (or conditional settings - not tested)
+        Test-Property -settingsDescription $settingsDescription -json $json -key 'workflowSchedule' -maynot
+        Test-Property -settingsDescription $settingsDescription -json $json -key 'workflowConcurrency' -maynot
+    }
     if ($type -eq 'Variable') {
         # Test for things that should / should not exist in a settings variable
     }
     if ($type -eq 'Project' -or $type -eq 'Workflow') {
         # templateUrl should not be in Project or Workflow settings
         Test-Property -settingsDescription $settingsDescription -json $json -key 'templateUrl' -maynot
-
-        # schedules and runs-on should not be in Project or Workflow settings
-        # These properties are used in Update AL-Go System Files, hence they should only be in Repo settings
-        'nextMajorSchedule','nextMinorSchedule','currentSchedule','runs-on' | ForEach-Object {
-            Test-Property -settingsDescription $settingsDescription -json $json -key $_ -shouldnot
-        }
     }
 }
 
@@ -100,7 +118,7 @@ function Test-JsonStr {
 
     try {
         $json = $jsonStr | ConvertFrom-Json | ConvertTo-HashTable
-        Test-SettingsJson -json $json -settingsDescription $settingsDescription -type:$type
+        Test-SettingsJson -json $json -settingsDescription $settingsDescription -type $type
     }
     catch {
         OutputError "$($_.Exception.Message.Replace("`r",'').Replace("`n",' ')) in $settingsDescription"

AL-Go-TestRepoHelper.ps1

@@ -50,16 +50,26 @@ function GetLatestTemplateSha {
     $branch = $templateUrl.Split('@')[1]
 
     try {
+        Write-Host "Get latest SHA for $templateUrl"
         $response = InvokeWebRequest -Headers $headers -Uri "$apiUrl/branches?per_page=100" -retry
-        $branchInfo = ($response.content | ConvertFrom-Json) | Where-Object { $_.Name -eq $branch }
     } catch {
-        if ($_.Exception.Message -like "*401*") {
-            throw "Failed to update AL-Go System Files. Make sure that the personal access token, defined in the secret called GhTokenWorkflow, is not expired and it has permission to update workflows. (Error was $($_.Exception.Message))"
-        } else {
+        Write-Host "Exception: $($_.Exception.Message)"
+        if ($_.Exception.Message -like "*Unauthorized*") {
+            try {
+                Write-Host "retry without Authorization header"
+                $headers.Remove('Authorization')
+                $response = InvokeWebRequest -Headers $headers -Uri "$apiUrl/branches?per_page=100" -retry
+            }
+            catch {
+                throw "Failed to update AL-Go System Files. Make sure that the personal access token, defined in the secret called GhTokenWorkflow, is not expired and it has permission to update workflows. (Error was $($_.Exception.Message))"
+            }
+        }
+        else {
             throw $_.Exception.Message
         }
     }
 
+    $branchInfo = ($response.content | ConvertFrom-Json) | Where-Object { $_.Name -eq $branch }
     if (!$branchInfo) {
         throw "$templateUrl doesn't exist"
     }
@@ -293,14 +303,53 @@ function GetWorkflowContentWithChangesFromSettings {
 
     $baseName = [System.IO.Path]::GetFileNameWithoutExtension($srcFile)
     $yaml = [Yaml]::Load($srcFile)
-    $workflowScheduleKey = "$($baseName)Schedule"
+    $yamlName = $yaml.get('name:')
+    if ($yamlName) {
+        $workflowName = $yamlName.content.SubString('name:'.Length).Trim().Trim('''"').Trim()
+    }
+    else {
+        $workflowName = $baseName
+    }
+
+    $workflowScheduleKey = "WorkflowSchedule"
+    $workflowConcurrencyKey = "WorkflowConcurrency"
+    foreach($key in @($workflowScheduleKey,$workflowConcurrencyKey)) {
+        if ($repoSettings.Keys -contains $key -and ($repoSettings."$key")) {
+            throw "The $key setting is not allowed in the global repository settings. Please use the workflow specific settings file or conditional settings."
+        }
+    }
 
-    # Any workflow (except for the PullRequestHandler and reusable workflows (_*)) can have a RepoSetting called <workflowname>Schedule, which will be used to set the schedule for the workflow
+    # Re-read settings and this time include workflow specific settings
+    $repoSettings = ReadSettings -buildMode '' -project '' -workflowName $workflowName -userName '' -branchName '' | ConvertTo-HashTable -recurse
+
+    # Old Schedule key is deprecated, but still supported
+    $oldWorkflowScheduleKey = "$($baseName)Schedule"
+    if ($repoSettings.Keys -contains $oldWorkflowScheduleKey) {
+        # DEPRECATION: REPLACE WITH ERROR AFTER October 1st 2025 --->
+        if ($repoSettings.Keys -contains $workflowScheduleKey) {
+            OutputWarning "Both $oldWorkflowScheduleKey and $workflowScheduleKey are defined in the settings file. $oldWorkflowScheduleKey will be ignored. This warning will become an error in the future"
+        }
+        else {
+            Trace-DeprecationWarning -Message "$oldWorkflowScheduleKey is deprecated" -DeprecationTag "_workflow_Schedule" -WillBecomeError
+            # Convert the old <workflow>Schedule setting to the new WorkflowSchedule setting
+            $repoSettings."$workflowScheduleKey" = @{ "cron" = $repoSettings."$oldWorkflowScheduleKey" }
+        }
+        # <--- REPLACE WITH ERROR AFTER October 1st 2025
+    }
+
+    # Any workflow (except for the PullRequestHandler and reusable workflows (_*)) can have concurrency and schedule defined
     if ($baseName -ne "PullRequestHandler" -and $baseName -notlike '_*') {
+        # Add Schedule and Concurrency settings to the workflow
         if ($repoSettings.Keys -contains $workflowScheduleKey) {
-            # Read the section under the on: key and add the schedule section
-            $yamlOn = $yaml.Get('on:/')
-            $yaml.Replace('on:/', $yamlOn.content+@('schedule:', "  - cron: '$($repoSettings."$workflowScheduleKey")'"))
+            if ($repoSettings."$workflowScheduleKey" -isnot [hashtable] -or $repoSettings."$workflowScheduleKey".Keys -notcontains 'cron' -or $repoSettings."$workflowScheduleKey".cron -isnot [string]) {
+                throw "The $workflowScheduleKey setting must be a structure containing a cron property"
+            }
+            # Replace or add the schedule part under the on: key
+            $yaml.ReplaceOrAdd('on:/', 'schedule:', @("- cron: '$($repoSettings."$workflowScheduleKey".cron)'"))
+        }
+        if ($repoSettings.Keys -contains $workflowConcurrencyKey) {
+            # Replace or add the concurrency part
+            $yaml.ReplaceOrAdd('', 'concurrency:', $repoSettings."$workflowConcurrencyKey")
         }
     }
 

CheckForUpdates/CheckForUpdates.HelperFunctions.ps1

@@ -16,27 +16,13 @@ Param(
 )
 
 . (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve)
+Import-Module (Join-Path -Path $PSScriptRoot -ChildPath '..\TelemetryHelper.psm1' -Resolve)
 . (Join-Path -Path $PSScriptRoot -ChildPath "yamlclass.ps1")
 . (Join-Path -Path $PSScriptRoot -ChildPath "CheckForUpdates.HelperFunctions.ps1")
 
 # ContainerHelper is used for determining project folders and dependencies
 DownloadAndImportBcContainerHelper
 
-if ($update -eq 'Y') {
-    if (-not $token) {
-        throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked."
-    }
-    else {
-        $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token))
-    }
-}
-
-# Use Authenticated API request to avoid the 60 API calls per hour limit
-$headers = @{
-    "Accept" = "application/vnd.github.baptiste-preview+json"
-    "Authorization" = "Bearer $token"
-}
-
 if (-not $templateUrl.Contains('@')) {
     $templateUrl += "@main"
 }
@@ -48,14 +34,36 @@ $templateUrl = $templateUrl -replace "^(https:\/\/)(www\.)(.*)$", '$1$3'
 
 # TemplateUrl is now always a full url + @ and a branch name
 
+if ($update -eq 'Y') {
+    if (-not $token) {
+        throw "The GhTokenWorkflow secret is needed. Read https://github.com/microsoft/AL-Go/blob/main/Scenarios/GhTokenWorkflow.md for more information."
+    }
+}
+
+$readToken = $token
+if ($token) {
+    # token comes from a secret, base 64 encoded
+    $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token))
+
+    # Get token with read permissions for this and the template repository - if private and in the same organization
+    $repositories = @($ENV:GITHUB_REPOSITORY)
+    if ($templateUrl -like "https://github.com/$($ENV:GITHUB_REPOSITORY_OWNER)/*") {
+        $repositories += $templateUrl.Split('@')[0]
+    }
+    $readToken = GetAccessToken -token $token -permissions @{"actions"="read";"contents"="read";"metadata"="read"} -repositories $repositories
+}
+
+# Use Authenticated API request if possible to avoid the 60 API calls per hour limit
+$headers = GetHeaders -token $readToken
+
 # CheckForUpdates will read all AL-Go System files from the Template repository and compare them to the ones in the current repository
 # CheckForUpdates will apply changes to the AL-Go System files based on AL-Go repo settings, such as "runs-on" etc.
 # if $update is set to Y, CheckForUpdates will also update the AL-Go System files in the current repository using a PR or a direct commit (if $directCommit is set to true)
 # if $update is set to N, CheckForUpdates will only check for updates and output a warning if there are updates available
 # if $downloadLatest is set to true, CheckForUpdates will download the latest version of the template repository, else it will use the templateSha setting in the .github/AL-Go-Settings file
 
 # Get Repo settings as a hashtable (do NOT read any specific project settings, nor any specific workflow, user or branch settings)
-$repoSettings = ReadSettings -project '' -workflowName '' -userName '' -branchName '' | ConvertTo-HashTable -recurse
+$repoSettings = ReadSettings -buildMode '' -project '' -workflowName '' -userName '' -branchName '' | ConvertTo-HashTable -recurse
 $templateSha = $repoSettings.templateSha
 $unusedALGoSystemFiles = $repoSettings.unusedALGoSystemFiles
 $includeBuildPP = $repoSettings.type -eq 'PTE' -and $repoSettings.powerPlatformSolutionFolder -ne ''
@@ -120,7 +128,7 @@ $removeFiles = @()
 $depth = 1
 if ($projects.Count -gt 1) {
     Import-Module (Join-Path -Path $PSScriptRoot -ChildPath "..\DetermineProjectsToBuild\DetermineProjectsToBuild.psm1" -Resolve) -DisableNameChecking
-    $allProjects, $projectsToBuild, $projectDependencies, $buildOrder = Get-ProjectsToBuild -baseFolder $baseFolder -buildAllProjects $true -maxBuildDepth 100
+    $allProjects, $modifiedProjects, $projectsToBuild, $projectDependencies, $buildOrder = Get-ProjectsToBuild -baseFolder $baseFolder -buildAllProjects $true -maxBuildDepth 100
     $depth = $buildOrder.Count
     Write-Host "Calculated dependency depth to be $depth"
 }
@@ -204,16 +212,21 @@ else {
     # $update set, update the files
     try {
         # If a pull request already exists with the same REF, then exit
-        $commitMessage = "[$updateBranch] Update AL-Go System Files from $templateInfo -  $templateSha"
-        $env:GH_TOKEN = $token
+        $branchSHA = RunAndCheck git rev-list -n 1 $updateBranch
+        $commitMessage = "[$($updateBranch)@$($branchSHA.SubString(0,7))] Update AL-Go System Files from $templateInfo - $($templateSha.SubString(0,7))"
+
+        # Get Token with permissions to modify workflows in this repository
+        $writeToken = GetAccessToken -token $token -permissions @{"actions"="read";"contents"="write";"pull_requests"="write";"workflows"="write"}
+        $env:GH_TOKEN = $writeToken
+
         $existingPullRequest = (gh api --paginate "/repos/$env:GITHUB_REPOSITORY/pulls?base=$updateBranch" -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" | ConvertFrom-Json) | Where-Object { $_.title -eq $commitMessage } | Select-Object -First 1
         if ($existingPullRequest) {
             OutputWarning "Pull request already exists for $($commitMessage): $($existingPullRequest.html_url)."
             exit
         }
 
         # If $directCommit, then changes are made directly to the default branch
-        $serverUrl, $branch = CloneIntoNewFolder -actor $actor -token $token -updateBranch $updateBranch -DirectCommit $directCommit -newBranchPrefix 'update-al-go-system-files'
+        $serverUrl, $branch = CloneIntoNewFolder -actor $actor -token $writeToken -updateBranch $updateBranch -DirectCommit $directCommit -newBranchPrefix 'update-al-go-system-files'
 
         invoke-git status
 
@@ -257,16 +270,16 @@ else {
         Write-Host "ReleaseNotes:"
         Write-Host $releaseNotes
 
-        if (!(CommitFromNewFolder -serverUrl $serverUrl -commitMessage $commitMessage -branch $branch -body $releaseNotes)) {
+        if (!(CommitFromNewFolder -serverUrl $serverUrl -commitMessage $commitMessage -branch $branch -body $releaseNotes -headBranch $updateBranch)) {
             OutputNotice -message "No updates available for AL-Go for GitHub."
         }
     }
     catch {
         if ($directCommit) {
-            throw "Failed to update AL-Go System Files. Make sure that the personal access token, defined in the secret called GhTokenWorkflow, is not expired and it has permission to update workflows. (Error was $($_.Exception.Message))"
+            throw "Failed to update AL-Go System Files. Make sure that the personal access token, defined in the secret called GhTokenWorkflow, is not expired and it has permission to update workflows. Read https://github.com/microsoft/AL-Go/blob/main/Scenarios/GhTokenWorkflow.md for more information. (Error was $($_.Exception.Message))"
         }
         else {
-            throw "Failed to create a pull-request to AL-Go System Files. Make sure that the personal access token, defined in the secret called GhTokenWorkflow, is not expired and it has permission to update workflows. (Error was $($_.Exception.Message))"
+            throw "Failed to create a pull-request to AL-Go System Files. Make sure that the personal access token, defined in the secret called GhTokenWorkflow, is not expired and it has permission to update workflows. Read https://github.com/microsoft/AL-Go/blob/main/Scenarios/GhTokenWorkflow.md for more information. (Error was $($_.Exception.Message))"
         }
     }
 }

CheckForUpdates/CheckForUpdates.ps1

@@ -144,9 +144,18 @@ class Yaml {
             $this.Remove($start, $count)
             $this.Insert($start, $yamlContent)
         }
-        else {
-            Write-Host -ForegroundColor Red "cannot locate $line"
-        }
+    }
+
+    # Add the lines in $content to the lines for the specified Yaml path, given by $line
+    [void] Add([string] $line, [string[]] $content) {
+        $this.Replace($line, $this.Get($line).content + $content)
+    }
+
+    # Replace or add a key and content to the lines for the specified Yaml path, given by $line
+    [void] ReplaceOrAdd([string] $line, [string] $key, [string[]]$content) {
+        # Remove the key part under the line
+        $this.Replace("$line$key",@())
+        $this.Add($line, @($key) + @($content | ForEach-Object { "  $_" }))
     }
 
     # Replace all occurrences of $from with $to throughout the Yaml content

CheckForUpdates/yamlclass.ps1

@@ -175,14 +175,15 @@ function NewSamplePerformanceTestApp
 
     UpdateManifest -sourceFolder $appSourceFolder -appJsonFile "$($destinationPath)\app.json" -name $name -publisher $publisher -idrange $idrange -version $version
 
+    $sampleRange = @{ "fromId" = 149100; "toId" = 149999 }
     if ($sampleCode) {
         Get-ChildItem -Path "$appSourceFolder\src" -Recurse -Filter "*.al" | ForEach-Object {
             Write-Host $_.Name
-            UpdateALFile -sourceFolder $_.DirectoryName -destinationFolder "$($destinationPath)\src" -alFileName $_.name -fromId 149100 -toId 149200 -startId $idrange[0]
+            UpdateALFile -sourceFolder $_.DirectoryName -destinationFolder "$($destinationPath)\src" -alFileName $_.name -fromId $sampleRange.fromId -toId $sampleRange.toId -startId $idrange[0]
         }
     }
     if ($sampleSuite) {
-        UpdateALFile -sourceFolder $alTemplatePath -destinationFolder $destinationPath -alFileName bcptSuite.json -fromId 149100 -toId 149200 -startId $idrange[0]
+        UpdateALFile -sourceFolder $alTemplatePath -destinationFolder $destinationPath -alFileName bcptSuite.json -fromId $sampleRange.fromId -toId $sampleRange.toId -startId $idrange[0]
     }
 }