From de9d2cadf92ee7ec71d80b03589d73c682b61b1c Mon Sep 17 00:00:00 2001 From: spetersenms Date: Thu, 6 Aug 2026 16:32:35 +0200 Subject: [PATCH 1/6] Add separate RunTests action gated by useSeparateTestAction Introduces a new opt-in RunTests action that moves normal test execution (testFolders) out of the RunPipeline action. When useSeparateTestAction is true, RunPipeline compiles, publishes and installs apps and keeps the build container alive, but skips normal tests; the RunTests action then runs those tests against the same container and produces TestResults.xml. BCPT and page scripting tests are unchanged. Behavior is unchanged when the setting is false. Includes the new setting (ReadSettings, settings.schema.json, settings.md), the RunPipeline keep-alive guard, template _BuildALGoProject wiring, release notes and Pester tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Actions/.Modules/ReadSettings.psm1 | 1 + Actions/.Modules/settings.schema.json | 4 + Actions/RunPipeline/RunPipeline.ps1 | 54 +++++ Actions/RunTests/README.md | 28 +++ Actions/RunTests/RunTests.ps1 | 102 +++++++++ Actions/RunTests/RunTests.psm1 | 160 ++++++++++++++ Actions/RunTests/action.yaml | 35 +++ RELEASENOTES.md | 4 + Scenarios/settings.md | 1 + .../.github/workflows/_BuildALGoProject.yaml | 13 +- .../.github/workflows/_BuildALGoProject.yaml | 13 +- Tests/RunTests.Action.Test.ps1 | 28 +++ Tests/RunTests.Test.ps1 | 209 ++++++++++++++++++ 13 files changed, 650 insertions(+), 2 deletions(-) create mode 100644 Actions/RunTests/README.md create mode 100644 Actions/RunTests/RunTests.ps1 create mode 100644 Actions/RunTests/RunTests.psm1 create mode 100644 Actions/RunTests/action.yaml create mode 100644 Tests/RunTests.Action.Test.ps1 create mode 100644 Tests/RunTests.Test.ps1 diff --git a/Actions/.Modules/ReadSettings.psm1 b/Actions/.Modules/ReadSettings.psm1 index 8109baf299..cefea3ee0b 100644 --- a/Actions/.Modules/ReadSettings.psm1 +++ b/Actions/.Modules/ReadSettings.psm1 @@ -174,6 +174,7 @@ function GetDefaultSettings "doNotBuildTests" = $false "doNotPerformUpgrade" = $false "doNotRunTests" = $false + "useSeparateTestAction" = $false "doNotRunBcptTests" = $false "doNotRunPageScriptingTests" = $false "doNotPublishApps" = $false diff --git a/Actions/.Modules/settings.schema.json b/Actions/.Modules/settings.schema.json index 4a1cd415d3..fe8e272767 100644 --- a/Actions/.Modules/settings.schema.json +++ b/Actions/.Modules/settings.schema.json @@ -344,6 +344,10 @@ "doNotRunTests": { "type": "boolean" }, + "useSeparateTestAction": { + "type": "boolean", + "description": "PREVIEW: When set to true, normal tests (testFolders) are no longer executed inside the RunPipeline action. Instead, RunPipeline keeps the build container alive and a separate RunTests action executes the tests afterwards. See https://aka.ms/ALGoSettings#useSeparateTestAction" + }, "doNotRunBcptTests": { "type": "boolean" }, diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index 07c125d2cf..a3c8bada72 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -19,6 +19,22 @@ Param( [string] $previousAppsPath = '' ) +function New-KeepAliveContainerCredential { + <# + .SYNOPSIS + Generates a credential used to create a build container that is kept alive for the RunTests action. + .DESCRIPTION + When useSeparateTestAction is enabled, RunPipeline keeps the build container alive so the RunTests + action can run tests against it. BcContainerHelper requires an explicit credential when a container + is kept (otherwise it is created with a random password that cannot be reused). This function returns + a PSCredential with a randomly generated complex password. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'A container password must be generated as plain text to build a reusable credential')] + param() + $password = "Pass!$([GUID]::NewGuid().ToString())" + return (New-Object pscredential 'admin', (ConvertTo-SecureString -String $password -AsPlainText -Force)) +} + $containerBaseFolder = $null $projectPath = $null @@ -473,6 +489,43 @@ try { $runAlPipelineParams["preprocessorsymbols"] = $settings.preprocessorSymbols $runAlPipelineParams["features"] = $settings.features + # When useSeparateTestAction is enabled, normal test execution is delegated to the + # separate RunTests action. Apps and test apps are still compiled, published and installed + # here, but the normal tests are not run (equivalent to doNotRunTests). The container is + # kept alive so the RunTests action can run the tests against it afterwards. + # This only affects normal tests; BCPT and page scripting tests are still run here. + # + # This only applies when Run-AlPipeline actually creates a build container to run tests against. + # A test-capable container is only created when apps are published (doNotPublishApps not set) and + # the build does not target an online environment. When apps are not published, Run-AlPipeline + # forces doNotRunTests and creates no test-capable container, so there is nothing for the RunTests + # action to hand off to - fall back to the normal RunPipeline behavior in that case. + $createsTestContainer = (-not $settings.doNotPublishApps) -and -not ($authContext -and $environmentName) + $keepContainerForSeparateTestAction = $false + if ($settings.useSeparateTestAction -and $createsTestContainer) { + Write-Host "useSeparateTestAction is enabled: skipping normal test execution in RunPipeline and keeping the container alive for the RunTests action" + $runAlPipelineParams["doNotRunTests"] = $true + $keepContainerForSeparateTestAction = $true + + # BcContainerHelper requires an explicit credential when the container is kept alive. Generate one + # here, pass it to Run-AlPipeline and surface it (masked, as base64-encoded JSON) to the RunTests + # action via the containerCredential environment variable so it can connect to the same container. + if (-not $runAlPipelineParams.ContainsKey('credential')) { + $containerCredential = New-KeepAliveContainerCredential + $runAlPipelineParams["credential"] = $containerCredential + + $containerCredentialPassword = $containerCredential.GetNetworkCredential().Password + $containerCredentialJson = @{ "username" = $containerCredential.UserName; "password" = $containerCredentialPassword } | ConvertTo-Json -Compress + $containerCredentialBase64 = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($containerCredentialJson)) + Write-Host "::add-mask::$containerCredentialPassword" + Write-Host "::add-mask::$containerCredentialBase64" + Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "containerCredential=$containerCredentialBase64" + } + } + elseif ($settings.useSeparateTestAction) { + Write-Host "::Notice::useSeparateTestAction is enabled, but no build container is created for this project (doNotPublishApps is set or the build targets an online environment), so the RunTests action has no container to run tests against and will be skipped." + } + Write-Host "Invoke Run-AlPipeline with buildmode $buildMode" Run-AlPipeline @runAlPipelineParams ` -accept_insiderEula ` @@ -518,6 +571,7 @@ try { -pageScriptingTestResultsFolder (Join-Path $buildArtifactFolder 'PageScriptingTestResultDetails') ` -CreateRuntimePackages:$CreateRuntimePackages ` -appVersion ($versionNumber.MajorMinorVersion) -appBuild ($versionNumber.BuildNumber) -appRevision ($versionNumber.RevisionNumber) ` + -keepContainer:$keepContainerForSeparateTestAction ` -uninstallRemovedApps if ($containerBaseFolder) { diff --git a/Actions/RunTests/README.md b/Actions/RunTests/README.md new file mode 100644 index 0000000000..12a4aa9877 --- /dev/null +++ b/Actions/RunTests/README.md @@ -0,0 +1,28 @@ +# Run tests + +Run the normal tests (testFolders) for an AL-Go project against the build container created and kept alive by the RunPipeline action. + +This action only does anything when the `useSeparateTestAction` setting is enabled. In that case, the RunPipeline action compiles, publishes and installs the apps, skips the normal tests and keeps the build container alive. This action then runs the normal tests against that same container and writes the results to `TestResults.xml` in the project folder. + +Only normal tests (testFolders) are handled here. BCPT and page scripting tests continue to be executed by the RunPipeline action. + +## INPUT + +### ENV variables + +| Name | Description | +| :-- | :-- | +| Settings | env.Settings must be set by a prior call to the ReadSettings Action | +| containerName | env.containerName is set by the RunPipeline action and identifies the container to run tests against (the container name is otherwise derived from the project) | + +### Parameters + +| Name | Required | Description | Default value | +| :-- | :-: | :-- | :-- | +| shell | | The shell (powershell or pwsh) in which the PowerShell script should run | powershell | +| project | | Project folder | '.' | +| installTestAppsJson | | Path to a JSON file containing a list of test apps to run tests in | '' | + +## OUTPUT + +None diff --git a/Actions/RunTests/RunTests.ps1 b/Actions/RunTests/RunTests.ps1 new file mode 100644 index 0000000000..4ce3f2458c --- /dev/null +++ b/Actions/RunTests/RunTests.ps1 @@ -0,0 +1,102 @@ +Param( + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', 'token', Justification = 'Exposed as $env:_token via action.yaml so downstream test override scripts (e.g. BCApps test tolerance artifact download) can authenticate.')] + [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] + [string] $token, + [Parameter(HelpMessage = "Project folder", Mandatory = $false)] + [string] $project = "", + [Parameter(HelpMessage = "A path to a JSON-formatted list of test apps to run tests in", Mandatory = $false)] + [string] $installTestAppsJson = '' +) + +<# +.SYNOPSIS + Runs the normal tests (testFolders) for an AL-Go project against the build container + created and kept alive by the RunPipeline action. +.DESCRIPTION + This action is the second half of the "split" between building/publishing apps and running + tests. It only does anything when the useSeparateTestAction setting is enabled. In that + case, RunPipeline compiles, publishes and installs the apps, skips the normal tests and + keeps the build container alive. This action then runs the normal tests against that same + container and writes the results to TestResults.xml in the project folder (the location the + AnalyzeTests action reads from). + + Only normal tests (testFolders) are handled here. BCPT and page scripting tests continue + to be executed by the RunPipeline action. +.PARAMETER token + The GitHub token running the action. It is exposed as the _token environment variable by + action.yaml so downstream test override scripts (for example, BCApps test tolerance, which + downloads the unstable-tests artifact) can authenticate against GitHub. +.PARAMETER project + Project folder. +.PARAMETER installTestAppsJson + A path to a JSON-formatted list of test apps (produced by previous jobs) to run tests in. +.EXAMPLE + RunTests.ps1 -project 'MyProject' +#> + +. (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve) +Import-Module (Join-Path $PSScriptRoot '..\TelemetryHelper.psm1' -Resolve) +Import-Module (Join-Path $PSScriptRoot 'RunTests.psm1' -Resolve) -DisableNameChecking -Force +DownloadAndImportBcContainerHelper + +function Get-TestRunnerCredential { + <# + .SYNOPSIS + Returns the credential used by the test runner to connect to the build container. + .DESCRIPTION + RunPipeline creates the container and keeps it alive when useSeparateTestAction is set. + When RunPipeline surfaces the container credential (masked, as base64-encoded JSON in + the containerCredential environment variable), it is used here so the test runner can + connect to the same container. Otherwise a default credential is used. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'The container credential is surfaced by RunPipeline as plain text')] + param() + if ($ENV:containerCredential) { + $credentialJson = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ENV:containerCredential)) | ConvertFrom-Json + $securePassword = ConvertTo-SecureString -String $credentialJson.password -AsPlainText -Force + return New-Object System.Management.Automation.PSCredential($credentialJson.username, $securePassword) + } + $securePassword = ConvertTo-SecureString -String ([GUID]::NewGuid().ToString()) -AsPlainText -Force + return New-Object System.Management.Automation.PSCredential("admin", $securePassword) +} + +if ($project -eq ".") { $project = "" } + +$baseFolder = $ENV:GITHUB_WORKSPACE +$projectPath = Join-Path $baseFolder $project + +Write-Host "Use settings" +$settings = $env:Settings | ConvertFrom-Json | ConvertTo-HashTable + +# This action is a no-op unless normal test execution has been delegated from RunPipeline via +# the useSeparateTestAction setting. In all other cases tests are executed inside RunPipeline. +if (-not $settings.useSeparateTestAction) { + Write-Host "useSeparateTestAction is not enabled. Tests are executed by the RunPipeline action. Skipping." + return +} + +# Analyze the repository to determine the test folders (and other test related settings) +$settings = AnalyzeRepo -settings $settings -baseFolder $baseFolder -project $project -doNotCheckArtifactSetting + +# Resolve the container kept alive by the RunPipeline action. +# The container name is deterministic per project and is also exported to the environment by RunPipeline. +$containerName = $ENV:containerName +if (-not $containerName) { + $containerName = GetContainerName($project) +} + +# Credentials used to connect to the build container. +$credential = Get-TestRunnerCredential + +# NOTE: RunTestsInBcContainer is the seam for the "local test runner". By default the +# BcContainerHelper test runner is used against the container created by RunPipeline; a +# custom/local test runner that behaves the same way can be provided as an override script. +$overrideParams = Get-ScriptOverrides -ALGoFolderName (Join-Path $projectPath ".AL-Go") -OverrideScriptNames @("RunTestsInBcContainer") + +Invoke-AlGoTestRun ` + -settings $settings ` + -projectPath $projectPath ` + -containerName $containerName ` + -credential $credential ` + -installTestAppsJson $installTestAppsJson ` + -runTestsOverride $overrideParams['RunTestsInBcContainer'] diff --git a/Actions/RunTests/RunTests.psm1 b/Actions/RunTests/RunTests.psm1 new file mode 100644 index 0000000000..96d63b8302 --- /dev/null +++ b/Actions/RunTests/RunTests.psm1 @@ -0,0 +1,160 @@ +<# +.SYNOPSIS + Helper module for the RunTests action. +.DESCRIPTION + Contains the logic for running the normal tests (testFolders) of an AL-Go project against a + build container that was created and kept alive by the RunPipeline action. Kept in a module + so the logic can be unit tested independently of the action entry script. +#> + +function Get-TestAppsToRun { + <# + .SYNOPSIS + Determines the set of test app files to run tests in. + .DESCRIPTION + Collects the test apps compiled for the project (found in the build artifacts TestApps + folder) and, when runTestsInAllInstalledTestApps is enabled, the test apps installed from + previous jobs (listed in installTestAppsJson). Test apps wrapped in parentheses are + unwrapped (matching Run-AlPipeline semantics where such apps are otherwise not tested). + .PARAMETER settings + The (analyzed) AL-Go settings hashtable. + .PARAMETER projectPath + The full path to the project folder. + .PARAMETER installTestAppsJson + Path to a JSON file with the list of installed test apps. + #> + Param( + [hashtable] $settings, + [string] $projectPath, + [string] $installTestAppsJson = '' + ) + + $testAppOutputFolder = Join-Path $projectPath ".buildartifacts\TestApps" + + $testApps = @() + if (Test-Path $testAppOutputFolder) { + $testApps += @(Get-ChildItem -Path $testAppOutputFolder -Filter "*.app" -ErrorAction SilentlyContinue | ForEach-Object { $_.FullName }) + } + + if ($settings.runTestsInAllInstalledTestApps -and $installTestAppsJson -and (Test-Path $installTestAppsJson)) { + try { + $installedTestApps = @(Get-Content -Path $installTestAppsJson -Raw | ConvertFrom-Json) + } + catch { + throw "Failed to parse JSON file at path '$installTestAppsJson'. Error: $($_.Exception.Message)" + } + $testApps += @($installedTestApps | ForEach-Object { $_.TrimStart("(").TrimEnd(")") } | Where-Object { $_ -and (Test-Path $_) }) + } + + return @($testApps | Select-Object -Unique) +} + +function Invoke-AlGoTestRun { + <# + .SYNOPSIS + Runs the normal tests for an AL-Go project against a kept-alive build container. + .DESCRIPTION + Runs tests in each test app against the given container and writes the results to + testResultsFile in JUnit format. Honors the doNotRunTests, doNotPublishApps and + treatTestFailuresAsWarnings settings (when doNotPublishApps is set, RunPipeline keeps no + container alive, so there is nothing to test against and this is a no-op). When a + RunTestsInBcContainer override is provided it is used instead of the built-in + BcContainerHelper test runner - this is the seam where a custom/local test runner + can be substituted. + .PARAMETER settings + The (analyzed) AL-Go settings hashtable. + .PARAMETER projectPath + The full path to the project folder. + .PARAMETER containerName + The name of the build container to run the tests against. + .PARAMETER credential + The credential used to connect to the build container. + .PARAMETER installTestAppsJson + Path to a JSON file with the list of installed test apps. + .PARAMETER runTestsOverride + Optional scriptblock overriding the BcContainerHelper test runner (RunTestsInBcContainer). + #> + Param( + [hashtable] $settings, + [string] $projectPath, + [string] $containerName, + [System.Management.Automation.PSCredential] $credential, + [string] $installTestAppsJson = '', + [scriptblock] $runTestsOverride = $null + ) + + if ($settings.doNotRunTests) { + Write-Host "doNotRunTests is set. Skipping test execution." + return + } + + if ($settings.doNotPublishApps) { + Write-Host "doNotPublishApps is set, so RunPipeline did not keep a build container alive. Skipping test execution." + return + } + + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installTestAppsJson + if ($testApps.Count -eq 0) { + Write-Host "No test apps found to run tests in. Skipping test execution." + return + } + + Write-Host "Running tests against container '$containerName'" + + $testResultsFile = Join-Path $projectPath "TestResults.xml" + if (Test-Path $testResultsFile) { + Remove-Item $testResultsFile -Force + } + + # GitHub Actions output severity for test failures. This mirrors how Run-AlPipeline configures + # Run-TestsInBcContainer: failing tests surface as warnings when treatTestFailuresAsWarnings is + # set, otherwise as errors. Valid values are 'no', 'error' and 'warning'. + $gitHubActionsSeverity = if ($settings.treatTestFailuresAsWarnings) { 'warning' } else { 'error' } + + $allTestsPassed = $true + Push-Location $projectPath + try { + foreach ($testApp in $testApps) { + $appJson = Get-AppJsonFromAppFile -appFile $testApp + Write-Host "Running tests in $($appJson.name) ($($appJson.id))" + + $runTestsParams = @{ + "containerName" = $containerName + "credential" = $credential + "companyName" = $settings.companyName + "extensionId" = $appJson.id + "appName" = $appJson.name + "JUnitResultFileName" = $testResultsFile + "AppendToJUnitResultFile" = $true + "detailed" = $true + "GitHubActions" = $gitHubActionsSeverity + "returnTrueIfAllPassed" = $true + } + + if ($runTestsOverride) { + $passed = & $runTestsOverride -parameters $runTestsParams + } + else { + $passed = Run-TestsInBcContainer @runTestsParams + } + + if (-not $passed) { + $allTestsPassed = $false + } + } + } + finally { + Pop-Location + } + + if (-not $allTestsPassed) { + if ($settings.treatTestFailuresAsWarnings) { + OutputWarning -message "There are test failures, but they are treated as warnings (treatTestFailuresAsWarnings is set)." + } + else { + throw "There are test failures." + } + } +} + +Export-ModuleMember -Function Invoke-AlGoTestRun, Get-TestAppsToRun diff --git a/Actions/RunTests/action.yaml b/Actions/RunTests/action.yaml new file mode 100644 index 0000000000..4cd99644f4 --- /dev/null +++ b/Actions/RunTests/action.yaml @@ -0,0 +1,35 @@ +name: Run Tests +author: Microsoft Corporation +inputs: + shell: + description: Shell in which you want to run the action (powershell or pwsh) + required: false + default: powershell + token: + description: The GitHub token running the action + required: false + default: ${{ github.token }} + project: + description: Project folder + required: false + default: '.' + installTestAppsJson: + description: A path to a JSON-formatted list of test apps to run tests in + required: false + default: '' +runs: + using: composite + steps: + - name: run + shell: ${{ inputs.shell }} + env: + _token: ${{ inputs.token }} + _project: ${{ inputs.project }} + _installTestAppsJson: ${{ inputs.installTestAppsJson }} + run: | + ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "RunTests" -Action { + ${{ github.action_path }}/RunTests.ps1 -token $ENV:_token -project $ENV:_project -installTestAppsJson $ENV:_installTestAppsJson + } +branding: + icon: terminal + color: blue diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 67f72ce759..fe02939b23 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,3 +1,7 @@ +### Separate test execution from RunPipeline (PREVIEW) + +A new `useSeparateTestAction` setting (default `false`) lets you move normal test execution (`testFolders`) out of the `RunPipeline` action and into a new dedicated `RunTests` action. When enabled, `RunPipeline` still compiles, publishes and installs the apps and keeps the build container alive, but does not run the normal tests. A new `RunTests` action then runs the tests against that same container and produces the same `TestResults.xml`. Only normal tests are affected; BCPT and page scripting tests are still executed by `RunPipeline`. When the setting is `false`, behavior is unchanged. + ### New `doNotPerformUpgrade` setting AL-Go now supports a new `doNotPerformUpgrade` setting that is passed through to `Run-AlPipeline`. Use it to skip the upgrade phase while still running the rest of the pipeline. diff --git a/Scenarios/settings.md b/Scenarios/settings.md index c0118f92d4..f24d63bcf1 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -244,6 +244,7 @@ Please read the release notes carefully when installing new versions of AL-Go fo | doNotBuildTests | This setting forces the pipeline to NOT build and run the tests and performance tests in testFolders and bcptTestFolders | false | | doNotRunTests | This setting forces the pipeline to NOT run the tests in testFolders. Tests are still being built and published. Note this setting can be set in a [workflow specific settings file](#where-are-the-settings-located) to only apply to that workflow | false | | doNotRunBcptTests | This setting forces the pipeline to NOT run the performance tests in testFolders. Performance tests are still being built and published. Note this setting can be set in a [workflow specific settings file](#where-are-the-settings-located) to only apply to that workflow | false | +| useSeparateTestAction | PREVIEW: When set to true, normal tests (testFolders) are no longer executed inside the RunPipeline action. Instead, RunPipeline still compiles, publishes and installs the apps and keeps the build container alive, and a separate RunTests action executes the tests afterwards against that container. This only affects normal tests; BCPT and page scripting tests are still executed by RunPipeline. Existing behavior is unchanged when this setting is false. | false | | memoryLimit | Specifies the memory limit for the build container. By default, this is left to BcContainerHelper to handle and will currently be set to 8G | 8G | | BcContainerHelperVersion | This setting can be set to a specific version (ex. 3.0.8) of BcContainerHelper to force AL-Go to use this version. **latest** means that AL-Go will use the latest released version. **preview** means that AL-Go will use the latest preview version. **dev** means that AL-Go will use the dev branch of containerhelper. | latest (or preview for AL-Go preview) | | unusedALGoSystemFiles (**deprecated**) | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | [ ] | diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index c93e03a6a8..a0eaa5e639 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -109,7 +109,7 @@ jobs: shell: ${{ inputs.shell }} project: ${{ inputs.project }} buildMode: ${{ inputs.buildMode }} - get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotRunBcptTests,doNotRunpageScriptingTests,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade + get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotRunBcptTests,doNotRunpageScriptingTests,useSeparateTestAction,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade - name: Run Build Initialize hook if: hashFiles(format('{0}/.AL-Go/BuildInitialize.ps1', inputs.project)) != '' @@ -228,6 +228,17 @@ jobs: baselineWorkflowSHA: ${{ inputs.baselineWorkflowSHA }} previousAppsPath: ${{ steps.DownloadPreviousRelease.outputs.PreviousAppsPath }} + - name: Run Tests + uses: microsoft/AL-Go-Actions/RunTests@main + if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && env.useSeparateTestAction == 'True' + env: + Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' + BuildMode: ${{ inputs.buildMode }} + with: + shell: ${{ inputs.shell }} + project: ${{ inputs.project }} + installTestAppsJson: ${{ steps.DownloadProjectDependencies.outputs.DownloadedTestApps }} + - name: Sign id: sign if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && inputs.signArtifacts && env.doNotSignApps == 'False' && (env.keyVaultCodesignCertificateName != '' || (fromJson(env.trustedSigning).Endpoint != '' && fromJson(env.trustedSigning).Account != '' && fromJson(env.trustedSigning).CertificateProfile != '')) && (hashFiles(format('{0}/.buildartifacts/Apps/*.app',inputs.project)) != '') diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index c93e03a6a8..a0eaa5e639 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -109,7 +109,7 @@ jobs: shell: ${{ inputs.shell }} project: ${{ inputs.project }} buildMode: ${{ inputs.buildMode }} - get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotRunBcptTests,doNotRunpageScriptingTests,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade + get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotRunBcptTests,doNotRunpageScriptingTests,useSeparateTestAction,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade - name: Run Build Initialize hook if: hashFiles(format('{0}/.AL-Go/BuildInitialize.ps1', inputs.project)) != '' @@ -228,6 +228,17 @@ jobs: baselineWorkflowSHA: ${{ inputs.baselineWorkflowSHA }} previousAppsPath: ${{ steps.DownloadPreviousRelease.outputs.PreviousAppsPath }} + - name: Run Tests + uses: microsoft/AL-Go-Actions/RunTests@main + if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && env.useSeparateTestAction == 'True' + env: + Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' + BuildMode: ${{ inputs.buildMode }} + with: + shell: ${{ inputs.shell }} + project: ${{ inputs.project }} + installTestAppsJson: ${{ steps.DownloadProjectDependencies.outputs.DownloadedTestApps }} + - name: Sign id: sign if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && inputs.signArtifacts && env.doNotSignApps == 'False' && (env.keyVaultCodesignCertificateName != '' || (fromJson(env.trustedSigning).Endpoint != '' && fromJson(env.trustedSigning).Account != '' && fromJson(env.trustedSigning).CertificateProfile != '')) && (hashFiles(format('{0}/.buildartifacts/Apps/*.app',inputs.project)) != '') diff --git a/Tests/RunTests.Action.Test.ps1 b/Tests/RunTests.Action.Test.ps1 new file mode 100644 index 0000000000..e9bdf19523 --- /dev/null +++ b/Tests/RunTests.Action.Test.ps1 @@ -0,0 +1,28 @@ +Get-Module TestActionsHelper | Remove-Module -Force +Import-Module (Join-Path $PSScriptRoot 'TestActionsHelper.psm1') +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 + +Describe "RunTests Action Tests" { + BeforeAll { + $actionName = "RunTests" + $scriptRoot = Join-Path $PSScriptRoot "..\Actions\$actionName" -Resolve + $scriptName = "$actionName.ps1" + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'scriptPath', Justification = 'False positive.')] + $scriptPath = Join-Path $scriptRoot $scriptName + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'actionScript', Justification = 'False positive.')] + $actionScript = GetActionScript -scriptRoot $scriptRoot -scriptName $scriptName + } + + It 'Compile Action' { + Invoke-Expression $actionScript + } + + It 'Test action.yaml matches script' { + $outputs = [ordered]@{ + } + YamlTest -scriptRoot $scriptRoot -actionName $actionName -actionScript $actionScript -outputs $outputs + } + + # Call action + +} diff --git a/Tests/RunTests.Test.ps1 b/Tests/RunTests.Test.ps1 new file mode 100644 index 0000000000..399ab5f296 --- /dev/null +++ b/Tests/RunTests.Test.ps1 @@ -0,0 +1,209 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'Mock/callback parameters must match function signatures')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'Test-only credential')] +param() + +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 + +. (Join-Path -Path $PSScriptRoot -ChildPath "../Actions/AL-Go-Helper.ps1" -Resolve) + +# Stub for the BcContainerHelper function so it can be mocked within the module scope +function Get-AppJsonFromAppFile { param($appFile) } + +Import-Module (Join-Path $PSScriptRoot '../Actions/RunTests/RunTests.psm1' -Resolve) -DisableNameChecking -Force + +Describe 'RunTests.psm1 Tests' { + BeforeAll { + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'testCredential', Justification = 'Used in tests')] + $testCredential = New-Object System.Management.Automation.PSCredential("admin", (ConvertTo-SecureString "password" -AsPlainText -Force)) + + function New-TestProject { + Param( + [string[]] $CompiledTestApps = @() + ) + $projectPath = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + $testAppsFolder = Join-Path $projectPath ".buildartifacts\TestApps" + New-Item -Path $testAppsFolder -ItemType Directory -Force | Out-Null + foreach ($app in $CompiledTestApps) { + New-Item -Path (Join-Path $testAppsFolder $app) -ItemType File -Force | Out-Null + } + return $projectPath + } + } + + Context 'Get-TestAppsToRun' { + It 'Collects compiled test apps from the build artifacts folder' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app', 'App2.Test.app') + $settings = @{ runTestsInAllInstalledTestApps = $false } + + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath + + $testApps.Count | Should -Be 2 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Includes installed test apps (unwrapping parentheses) when runTestsInAllInstalledTestApps is set' { + $projectPath = New-TestProject + $installedApp1 = Join-Path $projectPath 'Installed1.app' + $installedApp2 = Join-Path $projectPath 'Installed2.app' + New-Item -Path $installedApp1 -ItemType File -Force | Out-Null + New-Item -Path $installedApp2 -ItemType File -Force | Out-Null + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @($installedApp1, "($installedApp2)") | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ runTestsInAllInstalledTestApps = $true } + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson + + $testApps.Count | Should -Be 2 + $testApps | Should -Contain $installedApp1 + $testApps | Should -Contain $installedApp2 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Ignores installed test apps when runTestsInAllInstalledTestApps is not set' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $installedApp = Join-Path $projectPath 'Installed1.app' + New-Item -Path $installedApp -ItemType File -Force | Out-Null + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @($installedApp) | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ runTestsInAllInstalledTestApps = $false } + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson + + $testApps.Count | Should -Be 1 + Remove-Item -Path $projectPath -Recurse -Force + } + } + + Context 'Invoke-AlGoTestRun' { + It 'Does not run tests when doNotRunTests is set' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:runnerCalls = 0 + $override = { param($parameters) $script:runnerCalls++; return $true } + $settings = @{ doNotRunTests = $true; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:runnerCalls | Should -Be 0 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Does not run tests when doNotPublishApps is set (no container kept alive)' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:runnerCalls = 0 + $override = { param($parameters) $script:runnerCalls++; return $true } + $settings = @{ doNotRunTests = $false; doNotPublishApps = $true; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:runnerCalls | Should -Be 0 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Does not run tests when there are no test apps' { + $projectPath = New-TestProject + $script:runnerCalls = 0 + $override = { param($parameters) $script:runnerCalls++; return $true } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:runnerCalls | Should -Be 0 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Runs tests in every test app when tests pass' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app', 'App2.Test.app') + $script:runnerCalls = 0 + $override = { param($parameters) $script:runnerCalls++; return $true } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | Should -Not -Throw + + $script:runnerCalls | Should -Be 2 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Throws when a test fails and treatTestFailuresAsWarnings is not set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $override = { param($parameters) return $false } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | Should -Throw + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Does not throw when a test fails but treatTestFailuresAsWarnings is set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $override = { param($parameters) return $false } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $true } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | Should -Not -Throw + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Passes GitHubActions severity error when treatTestFailuresAsWarnings is not set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:capturedSeverity = $null + $override = { param($parameters) $script:capturedSeverity = $parameters.GitHubActions; return $true } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:capturedSeverity | Should -Be 'error' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Passes GitHubActions severity warning when treatTestFailuresAsWarnings is set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:capturedSeverity = $null + $override = { param($parameters) $script:capturedSeverity = $parameters.GitHubActions; return $true } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $true } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:capturedSeverity | Should -Be 'warning' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Builds a parameter set that is valid for the real Run-TestsInBcContainer cmdlet' { + # Guard against parameter drift: every key/value passed to the BcContainerHelper test + # runner is validated against the real cmdlet signature (parameter names and ValidateSet + # values). This catches invalid parameter names and out-of-set values locally instead of + # only surfacing them in CI, where the real cmdlet is actually invoked. + $command = Get-Command -Name 'Run-TestsInBcContainer' -ErrorAction SilentlyContinue + if (-not $command) { + Set-ItResult -Skipped -Because 'BcContainerHelper (Run-TestsInBcContainer) is not available in this environment' + return + } + if ($command.ResolvedCommand) { $command = $command.ResolvedCommand } + + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:capturedParams = $null + $override = { param($parameters) $script:capturedParams = $parameters; return $true } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:capturedParams | Should -Not -BeNullOrEmpty + foreach ($key in $script:capturedParams.Keys) { + $parameter = $command.Parameters[$key] + $parameter | Should -Not -BeNullOrEmpty -Because "'$key' must be a real parameter of Run-TestsInBcContainer" + + $validateSet = $parameter.Attributes | Where-Object { $_ -is [System.Management.Automation.ValidateSetAttribute] } | Select-Object -First 1 + if ($validateSet) { + $validateSet.ValidValues | Should -Contain $script:capturedParams[$key] -Because "the value for '$key' must be one of its allowed ValidateSet values" + } + } + + Remove-Item -Path $projectPath -Recurse -Force + } + } +} From 6550465e8e16d288c66273140419b9275fade06a Mon Sep 17 00:00:00 2001 From: spetersenms Date: Thu, 6 Aug 2026 17:32:51 +0200 Subject: [PATCH 2/6] Documentation cleanup and skip action when relevant. --- Actions/RunPipeline/RunPipeline.ps1 | 21 ++++++---------- Actions/RunTests/RunTests.ps1 | 25 ++++++++----------- Actions/RunTests/RunTests.psm1 | 11 +++----- .../.github/workflows/_BuildALGoProject.yaml | 4 +-- .../.github/workflows/_BuildALGoProject.yaml | 4 +-- 5 files changed, 24 insertions(+), 41 deletions(-) diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index a3c8bada72..34dcc1bf61 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -489,17 +489,11 @@ try { $runAlPipelineParams["preprocessorsymbols"] = $settings.preprocessorSymbols $runAlPipelineParams["features"] = $settings.features - # When useSeparateTestAction is enabled, normal test execution is delegated to the - # separate RunTests action. Apps and test apps are still compiled, published and installed - # here, but the normal tests are not run (equivalent to doNotRunTests). The container is - # kept alive so the RunTests action can run the tests against it afterwards. - # This only affects normal tests; BCPT and page scripting tests are still run here. - # - # This only applies when Run-AlPipeline actually creates a build container to run tests against. - # A test-capable container is only created when apps are published (doNotPublishApps not set) and - # the build does not target an online environment. When apps are not published, Run-AlPipeline - # forces doNotRunTests and creates no test-capable container, so there is nothing for the RunTests - # action to hand off to - fall back to the normal RunPipeline behavior in that case. + # When useSeparateTestAction is enabled, the normal tests (testFolders) are run by the separate + # RunTests action instead of here, and the build container is kept alive for it. BCPT and page + # scripting tests are unaffected. This needs a build container, which only exists when apps are + # published (doNotPublishApps not set) and the build does not target an online environment; + # otherwise the tests are run here as usual. $createsTestContainer = (-not $settings.doNotPublishApps) -and -not ($authContext -and $environmentName) $keepContainerForSeparateTestAction = $false if ($settings.useSeparateTestAction -and $createsTestContainer) { @@ -507,9 +501,8 @@ try { $runAlPipelineParams["doNotRunTests"] = $true $keepContainerForSeparateTestAction = $true - # BcContainerHelper requires an explicit credential when the container is kept alive. Generate one - # here, pass it to Run-AlPipeline and surface it (masked, as base64-encoded JSON) to the RunTests - # action via the containerCredential environment variable so it can connect to the same container. + # A kept-alive container needs an explicit credential so the RunTests action can reconnect to it. + # Generate one, pass it to Run-AlPipeline, and surface it (masked, base64 JSON) via containerCredential. if (-not $runAlPipelineParams.ContainsKey('credential')) { $containerCredential = New-KeepAliveContainerCredential $runAlPipelineParams["credential"] = $containerCredential diff --git a/Actions/RunTests/RunTests.ps1 b/Actions/RunTests/RunTests.ps1 index 4ce3f2458c..511ca8796e 100644 --- a/Actions/RunTests/RunTests.ps1 +++ b/Actions/RunTests/RunTests.ps1 @@ -13,15 +13,14 @@ Param( Runs the normal tests (testFolders) for an AL-Go project against the build container created and kept alive by the RunPipeline action. .DESCRIPTION - This action is the second half of the "split" between building/publishing apps and running - tests. It only does anything when the useSeparateTestAction setting is enabled. In that - case, RunPipeline compiles, publishes and installs the apps, skips the normal tests and - keeps the build container alive. This action then runs the normal tests against that same - container and writes the results to TestResults.xml in the project folder (the location the - AnalyzeTests action reads from). + Runs the normal tests (testFolders) of an AL-Go project. Tests are only run when the + useSeparateTestAction setting is enabled; otherwise this action does nothing and the tests are + run by the RunPipeline action. When enabled, RunPipeline compiles, publishes and installs the + apps and keeps the build container alive, and this action runs the normal tests against that + container and writes the results to TestResults.xml in the project folder. - Only normal tests (testFolders) are handled here. BCPT and page scripting tests continue - to be executed by the RunPipeline action. + Only normal tests (testFolders) are run here. BCPT and page scripting tests are run by the + RunPipeline action. .PARAMETER token The GitHub token running the action. It is exposed as the _token environment variable by action.yaml so downstream test override scripts (for example, BCApps test tolerance, which @@ -68,8 +67,7 @@ $projectPath = Join-Path $baseFolder $project Write-Host "Use settings" $settings = $env:Settings | ConvertFrom-Json | ConvertTo-HashTable -# This action is a no-op unless normal test execution has been delegated from RunPipeline via -# the useSeparateTestAction setting. In all other cases tests are executed inside RunPipeline. +# Tests only run here when useSeparateTestAction is enabled; otherwise RunPipeline runs them. if (-not $settings.useSeparateTestAction) { Write-Host "useSeparateTestAction is not enabled. Tests are executed by the RunPipeline action. Skipping." return @@ -78,8 +76,7 @@ if (-not $settings.useSeparateTestAction) { # Analyze the repository to determine the test folders (and other test related settings) $settings = AnalyzeRepo -settings $settings -baseFolder $baseFolder -project $project -doNotCheckArtifactSetting -# Resolve the container kept alive by the RunPipeline action. -# The container name is deterministic per project and is also exported to the environment by RunPipeline. +# Resolve the container kept alive by RunPipeline (name is deterministic per project, also exported to the environment). $containerName = $ENV:containerName if (-not $containerName) { $containerName = GetContainerName($project) @@ -88,9 +85,7 @@ if (-not $containerName) { # Credentials used to connect to the build container. $credential = Get-TestRunnerCredential -# NOTE: RunTestsInBcContainer is the seam for the "local test runner". By default the -# BcContainerHelper test runner is used against the container created by RunPipeline; a -# custom/local test runner that behaves the same way can be provided as an override script. +# A RunTestsInBcContainer override script, if present, replaces the built-in BcContainerHelper test runner. $overrideParams = Get-ScriptOverrides -ALGoFolderName (Join-Path $projectPath ".AL-Go") -OverrideScriptNames @("RunTestsInBcContainer") Invoke-AlGoTestRun ` diff --git a/Actions/RunTests/RunTests.psm1 b/Actions/RunTests/RunTests.psm1 index 96d63b8302..1f65c179d6 100644 --- a/Actions/RunTests/RunTests.psm1 +++ b/Actions/RunTests/RunTests.psm1 @@ -56,11 +56,8 @@ function Invoke-AlGoTestRun { .DESCRIPTION Runs tests in each test app against the given container and writes the results to testResultsFile in JUnit format. Honors the doNotRunTests, doNotPublishApps and - treatTestFailuresAsWarnings settings (when doNotPublishApps is set, RunPipeline keeps no - container alive, so there is nothing to test against and this is a no-op). When a - RunTestsInBcContainer override is provided it is used instead of the built-in - BcContainerHelper test runner - this is the seam where a custom/local test runner - can be substituted. + treatTestFailuresAsWarnings settings. When a RunTestsInBcContainer override script is + provided, it is used instead of the built-in BcContainerHelper test runner. .PARAMETER settings The (analyzed) AL-Go settings hashtable. .PARAMETER projectPath @@ -106,9 +103,7 @@ function Invoke-AlGoTestRun { Remove-Item $testResultsFile -Force } - # GitHub Actions output severity for test failures. This mirrors how Run-AlPipeline configures - # Run-TestsInBcContainer: failing tests surface as warnings when treatTestFailuresAsWarnings is - # set, otherwise as errors. Valid values are 'no', 'error' and 'warning'. + # Test failures surface as warnings when treatTestFailuresAsWarnings is set, otherwise as errors. $gitHubActionsSeverity = if ($settings.treatTestFailuresAsWarnings) { 'warning' } else { 'error' } $allTestsPassed = $true diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index a0eaa5e639..f46e1b86c8 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -109,7 +109,7 @@ jobs: shell: ${{ inputs.shell }} project: ${{ inputs.project }} buildMode: ${{ inputs.buildMode }} - get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotRunBcptTests,doNotRunpageScriptingTests,useSeparateTestAction,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade + get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotPublishApps,doNotRunBcptTests,doNotRunpageScriptingTests,useSeparateTestAction,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade - name: Run Build Initialize hook if: hashFiles(format('{0}/.AL-Go/BuildInitialize.ps1', inputs.project)) != '' @@ -230,7 +230,7 @@ jobs: - name: Run Tests uses: microsoft/AL-Go-Actions/RunTests@main - if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && env.useSeparateTestAction == 'True' + if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && env.useSeparateTestAction == 'True' && env.doNotPublishApps == 'False' && env.doNotRunTests == 'False' env: Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' BuildMode: ${{ inputs.buildMode }} diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index a0eaa5e639..f46e1b86c8 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -109,7 +109,7 @@ jobs: shell: ${{ inputs.shell }} project: ${{ inputs.project }} buildMode: ${{ inputs.buildMode }} - get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotRunBcptTests,doNotRunpageScriptingTests,useSeparateTestAction,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade + get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotPublishApps,doNotRunBcptTests,doNotRunpageScriptingTests,useSeparateTestAction,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade - name: Run Build Initialize hook if: hashFiles(format('{0}/.AL-Go/BuildInitialize.ps1', inputs.project)) != '' @@ -230,7 +230,7 @@ jobs: - name: Run Tests uses: microsoft/AL-Go-Actions/RunTests@main - if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && env.useSeparateTestAction == 'True' + if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && env.useSeparateTestAction == 'True' && env.doNotPublishApps == 'False' && env.doNotRunTests == 'False' env: Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' BuildMode: ${{ inputs.buildMode }} From cd2c03abcbc35fab56598f5fdda5f0c8d3266d1e Mon Sep 17 00:00:00 2001 From: spetersenms Date: Fri, 7 Aug 2026 10:22:33 +0200 Subject: [PATCH 3/6] Fix failing tests --- Actions/RunTests/RunTests.ps1 | 23 +++++++++-------------- Actions/RunTests/RunTests.psm1 | 18 ++++-------------- Tests/RunTests.Test.ps1 | 32 ++++---------------------------- 3 files changed, 17 insertions(+), 56 deletions(-) diff --git a/Actions/RunTests/RunTests.ps1 b/Actions/RunTests/RunTests.ps1 index 511ca8796e..2d0f86368b 100644 --- a/Actions/RunTests/RunTests.ps1 +++ b/Actions/RunTests/RunTests.ps1 @@ -1,5 +1,4 @@ Param( - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', 'token', Justification = 'Exposed as $env:_token via action.yaml so downstream test override scripts (e.g. BCApps test tolerance artifact download) can authenticate.')] [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] [string] $token, [Parameter(HelpMessage = "Project folder", Mandatory = $false)] @@ -13,18 +12,17 @@ Param( Runs the normal tests (testFolders) for an AL-Go project against the build container created and kept alive by the RunPipeline action. .DESCRIPTION - Runs the normal tests (testFolders) of an AL-Go project. Tests are only run when the - useSeparateTestAction setting is enabled; otherwise this action does nothing and the tests are - run by the RunPipeline action. When enabled, RunPipeline compiles, publishes and installs the - apps and keeps the build container alive, and this action runs the normal tests against that - container and writes the results to TestResults.xml in the project folder. + Runs the normal tests (testFolders) of an AL-Go project against the build container that the + RunPipeline action created and kept alive. This runs as part of the build when the + useSeparateTestAction setting is enabled; when it is not, RunPipeline runs the tests instead. + Results are written to TestResults.xml in the project folder. Only normal tests (testFolders) are run here. BCPT and page scripting tests are run by the RunPipeline action. .PARAMETER token - The GitHub token running the action. It is exposed as the _token environment variable by - action.yaml so downstream test override scripts (for example, BCApps test tolerance, which - downloads the unstable-tests artifact) can authenticate against GitHub. + The GitHub token running the action. It is exposed as the _token environment variable so + downstream test override scripts (for example, BCApps test tolerance, which downloads the + unstable-tests artifact) can authenticate against GitHub. .PARAMETER project Project folder. .PARAMETER installTestAppsJson @@ -67,11 +65,8 @@ $projectPath = Join-Path $baseFolder $project Write-Host "Use settings" $settings = $env:Settings | ConvertFrom-Json | ConvertTo-HashTable -# Tests only run here when useSeparateTestAction is enabled; otherwise RunPipeline runs them. -if (-not $settings.useSeparateTestAction) { - Write-Host "useSeparateTestAction is not enabled. Tests are executed by the RunPipeline action. Skipping." - return -} +# Surface the token so RunTestsInBcContainer override scripts (e.g. BCApps test tolerance) can authenticate against GitHub. +$ENV:_token = $token # Analyze the repository to determine the test folders (and other test related settings) $settings = AnalyzeRepo -settings $settings -baseFolder $baseFolder -project $project -doNotCheckArtifactSetting diff --git a/Actions/RunTests/RunTests.psm1 b/Actions/RunTests/RunTests.psm1 index 1f65c179d6..b06d01e3d1 100644 --- a/Actions/RunTests/RunTests.psm1 +++ b/Actions/RunTests/RunTests.psm1 @@ -55,9 +55,9 @@ function Invoke-AlGoTestRun { Runs the normal tests for an AL-Go project against a kept-alive build container. .DESCRIPTION Runs tests in each test app against the given container and writes the results to - testResultsFile in JUnit format. Honors the doNotRunTests, doNotPublishApps and - treatTestFailuresAsWarnings settings. When a RunTestsInBcContainer override script is - provided, it is used instead of the built-in BcContainerHelper test runner. + testResultsFile in JUnit format. Honors the treatTestFailuresAsWarnings setting. When a + RunTestsInBcContainer override script is provided, it is used instead of the built-in + BcContainerHelper test runner. .PARAMETER settings The (analyzed) AL-Go settings hashtable. .PARAMETER projectPath @@ -80,18 +80,8 @@ function Invoke-AlGoTestRun { [scriptblock] $runTestsOverride = $null ) - if ($settings.doNotRunTests) { - Write-Host "doNotRunTests is set. Skipping test execution." - return - } - - if ($settings.doNotPublishApps) { - Write-Host "doNotPublishApps is set, so RunPipeline did not keep a build container alive. Skipping test execution." - return - } - $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installTestAppsJson - if ($testApps.Count -eq 0) { + if (@($testApps).Count -eq 0) { Write-Host "No test apps found to run tests in. Skipping test execution." return } diff --git a/Tests/RunTests.Test.ps1 b/Tests/RunTests.Test.ps1 index 399ab5f296..89409e94df 100644 --- a/Tests/RunTests.Test.ps1 +++ b/Tests/RunTests.Test.ps1 @@ -37,7 +37,7 @@ Describe 'RunTests.psm1 Tests' { $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath - $testApps.Count | Should -Be 2 + @($testApps).Count | Should -Be 2 Remove-Item -Path $projectPath -Recurse -Force } @@ -53,7 +53,7 @@ Describe 'RunTests.psm1 Tests' { $settings = @{ runTestsInAllInstalledTestApps = $true } $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson - $testApps.Count | Should -Be 2 + @($testApps).Count | Should -Be 2 $testApps | Should -Contain $installedApp1 $testApps | Should -Contain $installedApp2 Remove-Item -Path $projectPath -Recurse -Force @@ -69,36 +69,12 @@ Describe 'RunTests.psm1 Tests' { $settings = @{ runTestsInAllInstalledTestApps = $false } $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson - $testApps.Count | Should -Be 1 + @($testApps).Count | Should -Be 1 Remove-Item -Path $projectPath -Recurse -Force } } Context 'Invoke-AlGoTestRun' { - It 'Does not run tests when doNotRunTests is set' { - $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') - $script:runnerCalls = 0 - $override = { param($parameters) $script:runnerCalls++; return $true } - $settings = @{ doNotRunTests = $true; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } - - Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override - - $script:runnerCalls | Should -Be 0 - Remove-Item -Path $projectPath -Recurse -Force - } - - It 'Does not run tests when doNotPublishApps is set (no container kept alive)' { - $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') - $script:runnerCalls = 0 - $override = { param($parameters) $script:runnerCalls++; return $true } - $settings = @{ doNotRunTests = $false; doNotPublishApps = $true; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } - - Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override - - $script:runnerCalls | Should -Be 0 - Remove-Item -Path $projectPath -Recurse -Force - } - It 'Does not run tests when there are no test apps' { $projectPath = New-TestProject $script:runnerCalls = 0 @@ -182,7 +158,7 @@ Describe 'RunTests.psm1 Tests' { Set-ItResult -Skipped -Because 'BcContainerHelper (Run-TestsInBcContainer) is not available in this environment' return } - if ($command.ResolvedCommand) { $command = $command.ResolvedCommand } + if (($command -is [System.Management.Automation.AliasInfo]) -and $command.ResolvedCommand) { $command = $command.ResolvedCommand } Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') From b015e221b9647c3e8dfdac212a19d30e78fe681e Mon Sep 17 00:00:00 2001 From: spetersenms Date: Fri, 7 Aug 2026 16:07:26 +0200 Subject: [PATCH 4/6] Use AlTool as the default test runner in the RunTests action The separate RunTests action now runs normal tests through Microsoft's headless al runtests (AlTool) CLI by default instead of BcContainerHelper. The runner resolves the kept-alive container's connection settings host-side from the container name, installs the AL developer tools as a dotnet global tool, runs the test codeunits batched (with an isolated fallback/rerun pass) and emits the same TestResults.xml (JUnit) schema, so downstream test result analysis is unchanged. No RunPipeline change is required. BcContainerHelper remains available via a RunTestsInBcContainer override script, which fully replaces the built-in AlTool runner (this is how, for example, BCApps supplies its own runner and how Legacy test-type tests are handled). The AlTool runner does not run Legacy test types or tests that require UI/client-callback interaction; this is documented as a known limitation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Actions/RunTests/AlToolTestRunner.psm1 | 923 +++++++++++++++++++++++++ Actions/RunTests/README.md | 10 + Actions/RunTests/RunTests.ps1 | 2 +- Actions/RunTests/RunTests.psm1 | 16 +- RELEASENOTES.md | 5 + Tests/AlToolTestRunner.Test.ps1 | 260 +++++++ Tests/RunTests.Test.ps1 | 50 ++ 7 files changed, 1260 insertions(+), 6 deletions(-) create mode 100644 Actions/RunTests/AlToolTestRunner.psm1 create mode 100644 Tests/AlToolTestRunner.Test.ps1 diff --git a/Actions/RunTests/AlToolTestRunner.psm1 b/Actions/RunTests/AlToolTestRunner.psm1 new file mode 100644 index 0000000000..910492b1f4 --- /dev/null +++ b/Actions/RunTests/AlToolTestRunner.psm1 @@ -0,0 +1,923 @@ +<# +.SYNOPSIS + Built-in test runner that drives Microsoft's headless `al runtests` (altool) CLI instead of + BcContainerHelper's client-session runner, while producing the same JUnit XML the AL-Go + pipeline already consumes downstream. + +.DESCRIPTION + This module is the default test runner used by the RunTests action (Invoke-AlGoTestRun) when no + RunTestsInBcContainer override script is supplied. For a single test app (identified by + extensionId) it: + 1. Ensures the `al` CLI is installed (dotnet global tool, prerelease). + 2. Resolves the kept-alive container's on-prem connection settings (server/instance/port) + host-side from the container name via BcContainerHelper, and generates a throw-away AL + project with a launch.json so `al runtests` has connection settings. + 3. Enumerates the app's test codeunits + methods via Get-TestsFromBcContainer. + 4. Runs `al runtests` batched in one `--testplan` invocation (single auth/connect), with an + isolated per-codeunit fallback/rerun pass for any codeunit that produced no result or a + failure. + 5. Emits a JUnit results file matching the exact schema BcContainerHelper produces, so the + downstream AnalyzeTests step keeps working unchanged. + + Credentials are taken from the parameters' PSCredential and exposed to `al` through the + BC_SERVER_USERNAME / BC_SERVER_PASSWORD environment variables (the only auth mechanism the CLI + supports for on-prem UserPassword). + + Known altool output quirks handled here: + - The Results: block emits a phantom `PASS OnRun (..)` trigger entry and a trailing + empty-named aggregate entry per codeunit; both are dropped so counts match the real methods. + - Failure text is the indented lines after `FAIL (Nms)` up to `AL Callstack:`; the + callstack follows until the next result line. +#> + +$ErrorActionPreference = "Stop" + +$script:AlToolPackageId = "Microsoft.Dynamics.BusinessCentral.Development.Tools" + +<# +.SYNOPSIS + Ensures the `al` CLI is available on PATH, installing the prerelease dotnet global tool. +.DESCRIPTION + Installs (or, when already present, leaves in place) the AL developer tools as a dotnet global + tool. The install is guarded by a named mutex so concurrent jobs on the same runner do not + collide on the shared tools store, and availability is re-checked after acquiring the mutex. +.OUTPUTS + [string] The resolved `al` version string. +#> +function Install-AlTool { + param( + [switch] $Force + ) + + $toolsPath = Join-Path $env:USERPROFILE ".dotnet\tools" + if ($env:HOME -and -not $env:USERPROFILE) { + $toolsPath = Join-Path $env:HOME ".dotnet/tools" + } + if (($env:PATH -split [System.IO.Path]::PathSeparator) -notcontains $toolsPath) { + $env:PATH = "$env:PATH$([System.IO.Path]::PathSeparator)$toolsPath" + } + + # Serialize install/update across processes with a named mutex and re-check availability after + # acquiring it (another job may have just installed it). + $mutex = New-Object System.Threading.Mutex($false, "Global\AL-Go-AlTool-Install") + $acquired = $false + try { + try { $acquired = $mutex.WaitOne([TimeSpan]::FromMinutes(10)) } catch [System.Threading.AbandonedMutexException] { $acquired = $true } + + $alAvailable = $null -ne (Get-Command al -ErrorAction SilentlyContinue) + + if (-not $alAvailable) { + Write-Host "Installing '$script:AlToolPackageId' (prerelease) as a dotnet global tool..." + & dotnet tool install $script:AlToolPackageId --global --prerelease *>&1 | ForEach-Object { Write-Host $_ } + if ($LASTEXITCODE -ne 0) { + # A concurrent job may have installed it first; treat as success if `al` now resolves, + # otherwise fall back to an update. + if ($null -eq (Get-Command al -ErrorAction SilentlyContinue)) { + & dotnet tool update $script:AlToolPackageId --global --prerelease *>&1 | ForEach-Object { Write-Host $_ } + } + } + } + elseif ($Force) { + # Explicit opt-in moves to the newest prerelease once, under the mutex. + try { + & dotnet tool update $script:AlToolPackageId --global --prerelease *>&1 | ForEach-Object { Write-Host $_ } + } + catch { + Write-Host "WARNING: 'al' update check failed ($($_.Exception.Message)). Using existing version." + } + } + } + finally { + if ($acquired) { $mutex.ReleaseMutex() } + $mutex.Dispose() + } + + if (-not (Get-Command al -ErrorAction SilentlyContinue)) { + throw "The 'al' CLI is not available after installation. Ensure '$toolsPath' is on PATH and that the runner can reach nuget.org." + } + + $version = (& al --version 2>&1 | Select-Object -First 1) + Write-Host "Using al CLI version: $version" + return "$version" +} + +<# +.SYNOPSIS + Resolves the on-prem connection settings (server URL, instance, dev-service port) for a container. +.DESCRIPTION + BcContainerHelper's Run-TestsInBcContainer only needs the container name and resolves the + endpoint internally, but `al runtests` needs an explicit server/instance/port. This reads them + host-side from the container's server configuration, falling back to conventional defaults. +.PARAMETER ContainerName + The name of the build container. +.OUTPUTS + [hashtable] @{ Server; ServerInstance; Port } +#> +function Get-AlToolConnection { + param( + [Parameter(Mandatory = $true)][string] $ContainerName + ) + + $server = "http://$ContainerName" + $instance = "BC" + $port = 7049 + + try { + $config = Get-BcContainerServerConfiguration -ContainerName $ContainerName + if ($config) { + if ($config.ServerInstance) { $instance = "$($config.ServerInstance)" } + if ($config.DeveloperServicesPort) { $port = [int]$config.DeveloperServicesPort } + } + } + catch { + Write-Host "WARNING: Could not read server configuration for '$ContainerName' ($($_.Exception.Message)). Falling back to $server/${instance}:$port." + } + + return @{ Server = $server; ServerInstance = $instance; Port = $port } +} + +<# +.SYNOPSIS + Creates a throw-away AL project folder with a launch.json targeting the container so `al runtests` + can resolve connection settings. +.PARAMETER ContainerName + The name of the build container. +.PARAMETER Tenant + The tenant to connect to. +.PARAMETER Connection + The connection hashtable produced by Get-AlToolConnection. +.OUTPUTS + [string] Path to the generated project folder. +#> +function New-AlToolProject { + param( + [Parameter(Mandatory = $true)][string] $ContainerName, + [Parameter(Mandatory = $true)][string] $Tenant, + [Parameter(Mandatory = $true)][hashtable] $Connection + ) + + $projectRoot = Join-Path ([System.IO.Path]::GetTempPath()) "altool-project-$ContainerName" + $vscodeDir = Join-Path $projectRoot ".vscode" + New-Item -ItemType Directory -Path $vscodeDir -Force | Out-Null + + $appJson = [ordered]@{ + id = [System.Guid]::NewGuid().ToString() + name = "AlToolTestDriver" + publisher = "AL-Go" + version = "1.0.0.0" + platform = "1.0.0.0" + runtime = "15.0" + } + $appJson | ConvertTo-Json | Set-Content -Path (Join-Path $projectRoot "app.json") -Encoding UTF8 + + $launch = [ordered]@{ + version = "0.2.0" + configurations = @( + [ordered]@{ + name = "altool" + type = "al" + request = "launch" + server = $Connection.Server + serverInstance = $Connection.ServerInstance + port = $Connection.Port + tenant = $Tenant + authentication = "UserPassword" + environmentType = "OnPrem" + startupObjectId = 22 + startupObjectType = "Page" + schemaUpdateMode = "Synchronize" + } + ) + } + $launch | ConvertTo-Json -Depth 5 | Set-Content -Path (Join-Path $vscodeDir "launch.json") -Encoding UTF8 + + return $projectRoot +} + +<# +.SYNOPSIS + Resolves the company `al runtests` should target. +.DESCRIPTION + Honors an explicitly requested company name first, then falls back to the container's default + (preferring an evaluation company) via Get-CompanyInBcContainer. +.PARAMETER ContainerName + The name of the build container. +.PARAMETER Tenant + The tenant to connect to. +.PARAMETER CompanyName + The company name requested by the caller (optional). +.OUTPUTS + [string] Company name, or empty string if none could be resolved. +#> +function Get-AlToolCompany { + param( + [Parameter(Mandatory = $true)][string] $ContainerName, + [Parameter(Mandatory = $true)][string] $Tenant, + [string] $CompanyName = "" + ) + + if (-not [string]::IsNullOrWhiteSpace($CompanyName)) { + return $CompanyName + } + + try { + $companies = @(Get-CompanyInBcContainer -containerName $ContainerName -tenant $Tenant) + if ($companies.Count -gt 0) { + $preferred = $companies | Where-Object { $_.evaluationCompany -eq $true } | Select-Object -First 1 + $company = if ($preferred) { $preferred.companyName } else { $companies[0].companyName } + return "$company" + } + } + catch { + Write-Host "WARNING: Could not enumerate companies for '$ContainerName' ($($_.Exception.Message))." + } + return "" +} + +<# +.SYNOPSIS + Builds disabled-test lookups from the disabledTests list: per-method keys plus whole-codeunit + names (where a `*` wildcard method disables the entire codeunit). +.DESCRIPTION + Each disabledTests entry has a codeunitName and either a single 'method', an array of methods, or + the wildcard '*'. A '*' entry means the ENTIRE codeunit is disabled, so it must exclude every + method of that codeunit - not a literal method named '*'. Names/keys are lowercased for + case-insensitive matching. +.PARAMETER DisabledTests + Array of disabled-test entries. +.OUTPUTS + [hashtable] @{ Methods = ::">; Codeunits = "> } +#> +function Get-DisabledTestKeySet { + param( + [array] $DisabledTests = @() + ) + + $methodSet = @{} + $codeunitSet = @{} + foreach ($entry in $DisabledTests) { + if (-not $entry) { continue } + $cuName = "$($entry.codeunitName)".ToLowerInvariant() + $methods = @() + if ($entry.PSObject.Properties['method'] -and $entry.method) { $methods = @($entry.method) } + foreach ($m in $methods) { + if ("$m" -eq '*') { + $codeunitSet[$cuName] = $true + } + else { + $methodSet["$cuName::$("$m".ToLowerInvariant())"] = $true + } + } + } + return @{ Methods = $methodSet; Codeunits = $codeunitSet } +} + +<# +.SYNOPSIS + Enumerates the test codeunits + enabled methods for an app in the container. +.DESCRIPTION + Uses Get-TestsFromBcContainer with the app's extensionId to list every test codeunit and method. + When a disabledTests list is supplied, disabled methods (and whole codeunits marked with a `*` + wildcard) are filtered out here, because `al runtests` has no equivalent of BCH's run-time + DisableTestMethod control. +.PARAMETER Parameters + Hashtable with containerName, tenant, credential, extensionId and optionally testType and + disabledTests. +.OUTPUTS + [object[]] Codeunit objects with .Id, .Name, .Tests (enabled method name array). +#> +function Get-AlToolTestCodeunits { + param( + [Parameter(Mandatory = $true)][hashtable] $Parameters + ) + + $getTestsParams = @{ + containerName = $Parameters.containerName + tenant = if ($Parameters.ContainsKey("tenant") -and $Parameters.tenant) { $Parameters.tenant } else { "default" } + credential = $Parameters.credential + extensionId = $Parameters.extensionId + ignoreGroups = $true + } + if ($Parameters.ContainsKey("testType") -and $Parameters.testType) { + $getTestsParams.testType = $Parameters.testType + } + + $codeunits = @(Get-TestsFromBcContainer @getTestsParams) + + $disabledMethods = @{} + $disabledCodeunits = @{} + if ($Parameters.ContainsKey("disabledTests") -and $Parameters.disabledTests) { + $lookup = Get-DisabledTestKeySet -DisabledTests @($Parameters.disabledTests) + $disabledMethods = $lookup.Methods + $disabledCodeunits = $lookup.Codeunits + } + + $result = @() + $disabledCount = 0 + foreach ($cu in $codeunits) { + $cuNameLower = "$($cu.Name)".ToLowerInvariant() + $methods = @($cu.Tests | ForEach-Object { "$_" }) + + if ($disabledCodeunits.ContainsKey($cuNameLower)) { + $disabledCount += $methods.Count + continue + } + + if ($disabledMethods.Count -gt 0) { + $enabled = @($methods | Where-Object { -not $disabledMethods.ContainsKey("$cuNameLower::$("$_".ToLowerInvariant())") }) + $disabledCount += ($methods.Count - $enabled.Count) + $methods = $enabled + } + if ($methods.Count -gt 0) { + $result += [PSCustomObject]@{ Id = $cu.Id; Name = $cu.Name; Tests = $methods } + } + } + + if ($disabledCount -gt 0) { + Write-Host "Excluded $disabledCount disabled test method(s) from altool enumeration." + } + return @($result) +} + +<# +.SYNOPSIS + Parses the `Results:` block of a single `al runtests` invocation into per-method outcomes. +.DESCRIPTION + Drops the phantom `OnRun` trigger entry and the trailing empty-named aggregate entry. Captures + the failure message (lines up to `AL Callstack:`) and callstack (following lines) for failures. +.PARAMETER OutputLines + The lines of `al runtests --raw` output. +.OUTPUTS + [hashtable] method name -> @{ Outcome (Pass/Fail/Skip); Ms; Message; Stacktrace } +#> +function ConvertFrom-AlRunTestsOutput { + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][AllowEmptyString()][string[]] $OutputLines + ) + + $results = @{} + $resultLineRegex = '^\s*(PASS|FAIL|SKIP)\s+(.*?)\s*\((\d+)ms\)\s*$' + + $startIdx = -1 + for ($i = 0; $i -lt $OutputLines.Count; $i++) { + if ($OutputLines[$i] -match '^\s*Results:\s*$') { $startIdx = $i + 1; break } + } + if ($startIdx -lt 0) { return $results } + + $i = $startIdx + while ($i -lt $OutputLines.Count) { + $line = $OutputLines[$i] + $m = [regex]::Match($line, $resultLineRegex) + if (-not $m.Success) { $i++; continue } + + $outcome = switch ($m.Groups[1].Value) { 'PASS' { 'Pass' } 'FAIL' { 'Fail' } 'SKIP' { 'Skip' } } + $name = $m.Groups[2].Value.Trim() + $ms = [int]$m.Groups[3].Value + + if ([string]::IsNullOrWhiteSpace($name) -or $name -eq 'OnRun') { $i++; continue } + + $message = '' + $stackText = '' + if ($outcome -eq 'Fail') { + $msgLines = @() + $stackLines = @() + $inStack = $false + $j = $i + 1 + while ($j -lt $OutputLines.Count) { + $next = $OutputLines[$j] + if ([regex]::IsMatch($next, $resultLineRegex)) { break } + if ($next -match '^\s*AL Callstack:\s*$') { $inStack = $true; $j++; continue } + if ($inStack) { + if ($next.Trim().Length -gt 0) { $stackLines += $next.Trim() } + } + else { + if ($next.Trim().Length -gt 0) { $msgLines += $next.Trim() } + } + $j++ + } + $message = ($msgLines -join ' ').Trim() + $stackText = ($stackLines -join ';') + $i = $j + } + else { + $i++ + } + + $results[$name] = @{ Outcome = $outcome; Ms = $ms; Message = $message; Stacktrace = $stackText } + } + + return $results +} + +<# +.SYNOPSIS + Splits batched `al runtests --testplan --raw` output into per-codeunit result maps. +.DESCRIPTION + The batched runner emits one block per codeunit, each preceded by a "===== Codeunit =====" + marker and containing a "Results:" section. This splits on the marker and parses each block with + ConvertFrom-AlRunTestsOutput, keyed by codeunit id. +.PARAMETER OutputLines + The lines of batched `al runtests --raw` output. +.OUTPUTS + [hashtable] "" -> (method-name -> @{ Outcome; Ms; Message; Stacktrace }) +#> +function ConvertFrom-AlBatchOutput { + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][AllowEmptyString()][string[]] $OutputLines + ) + + $byCodeunit = @{} + $markerRegex = '^\s*=====\s*Codeunit\s+(\d+)\s*=====\s*$' + + $currentId = $null + $currentLines = New-Object System.Collections.Generic.List[string] + + $flush = { + if ($null -ne $currentId) { + $byCodeunit["$currentId"] = ConvertFrom-AlRunTestsOutput -OutputLines @($currentLines.ToArray()) + } + } + + foreach ($line in $OutputLines) { + $m = [regex]::Match($line, $markerRegex) + if ($m.Success) { + & $flush + $currentId = $m.Groups[1].Value + $currentLines = New-Object System.Collections.Generic.List[string] + continue + } + if ($null -ne $currentId) { $currentLines.Add($line) } + } + & $flush + + return $byCodeunit +} + +<# +.SYNOPSIS + Serializes a batch of test groups into the JSON array the al `--testplan` option expects. +.DESCRIPTION + Built by hand (not ConvertTo-Json) because PowerShell's ConvertTo-Json collapses single-element + arrays to a scalar/object - a one-codeunit plan would become {..} not [{..}], and a one-method + list "M" not ["M"] - which the al tool rejects. This guarantees arrays at both levels for any count. +.PARAMETER Groups + Array of @{ Id; Methods } describing the codeunits (and enabled methods) to run. +.OUTPUTS + [string] JSON array: [{ "codeunitId": N, "testMethods": [ ... ] }, ...] +#> +function ConvertTo-AlTestPlanJson { + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]] $Groups + ) + + $sb = New-Object System.Text.StringBuilder + [void]$sb.Append('[') + $firstGroup = $true + foreach ($g in $Groups) { + if (-not $firstGroup) { [void]$sb.Append(',') } + $firstGroup = $false + $methodsJson = @($g.Methods | ForEach-Object { "$_" | ConvertTo-Json -Compress }) + [void]$sb.Append(('{{"codeunitId":{0},"testMethods":[{1}]}}' -f [int]$g.Id, ($methodsJson -join ','))) + } + [void]$sb.Append(']') + return $sb.ToString() +} + +<# +.SYNOPSIS + Runs `al runtests` for one codeunit and returns the parsed per-method results plus raw output. +.PARAMETER CodeunitId + The codeunit id to run. +.PARAMETER Methods + The test method names to run. +.PARAMETER ProjectPath + The throw-away AL project folder. +.PARAMETER Company + The company to run against. +.PARAMETER Tenant + The tenant to connect to. +.PARAMETER Connection + The connection hashtable produced by Get-AlToolConnection. +.OUTPUTS + [hashtable] @{ Results (method->outcome map); ElapsedSec; Raw; Connected (bool) } +#> +function Invoke-AlRunTestsForCodeunit { + param( + [Parameter(Mandatory = $true)][string] $CodeunitId, + [Parameter(Mandatory = $true)][string[]] $Methods, + [Parameter(Mandatory = $true)][string] $ProjectPath, + [Parameter(Mandatory = $true)][string] $Company, + [Parameter(Mandatory = $true)][string] $Tenant, + [Parameter(Mandatory = $true)][hashtable] $Connection + ) + + # `al runtests` emits structured JSON by default in newer builds; `--raw` restores the + # human-readable text summary ("Test run completed: ..." + a "Results:" block of + # "PASS|FAIL|SKIP (Nms)" lines) that ConvertFrom-AlRunTestsOutput parses. + $alArgs = @( + 'runtests', $CodeunitId, + '--project', $ProjectPath, + '--company', $Company, + '--server', $Connection.Server, + '--serverinstance', $Connection.ServerInstance, + '--port', "$($Connection.Port)", + '--environmenttype', 'OnPrem', + '--authentication', 'UserPassword', + '--tenant', $Tenant, + '--raw', + '--testmethods' + ) + $Methods + + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $output = & al @alArgs 2>&1 + $sw.Stop() + + $lines = @($output | ForEach-Object { "$_" }) + $connected = ($lines | Where-Object { $_ -match 'Test run completed:' }).Count -gt 0 + $parsed = ConvertFrom-AlRunTestsOutput -OutputLines $lines + + if ($connected -and $parsed.Count -eq 0 -and $Methods.Count -gt 0) { + Write-Host "::warning::al runtests connected for codeunit $CodeunitId but produced no parseable results. Raw output follows (possible output-format change):" + Write-Host ($lines -join "`n") + } + + return @{ + Results = $parsed + ElapsedSec = [Math]::Round($sw.Elapsed.TotalSeconds, 3) + Raw = ($lines -join "`n") + Connected = $connected + } +} + +<# +.SYNOPSIS + Runs a batch of codeunits in ONE `al runtests --testplan` invocation (single connection + auth + + session) and returns per-codeunit results. This removes the per-codeunit connect/auth tax. +.PARAMETER Groups + Array of @{ Id; Methods } describing the codeunits (and enabled methods) to run. +.PARAMETER ProjectPath + The throw-away AL project folder. +.PARAMETER Company + The company to run against. +.PARAMETER Tenant + The tenant to connect to. +.PARAMETER Connection + The connection hashtable produced by Get-AlToolConnection. +.OUTPUTS + [hashtable] @{ Results (""->method map); ElapsedSec; Connected; Raw } +#> +function Invoke-AlBatchRunTests { + param( + [Parameter(Mandatory = $true)][object[]] $Groups, + [Parameter(Mandatory = $true)][string] $ProjectPath, + [Parameter(Mandatory = $true)][string] $Company, + [Parameter(Mandatory = $true)][string] $Tenant, + [Parameter(Mandatory = $true)][hashtable] $Connection + ) + + # Write the test plan as a JSON file (avoids command-line length limits with many codeunits). + $planFile = Join-Path ([System.IO.Path]::GetTempPath()) ("altool-plan-" + [System.Guid]::NewGuid().ToString('N') + ".json") + Set-Content -Path $planFile -Value (ConvertTo-AlTestPlanJson -Groups $Groups) -Encoding UTF8 + + try { + $alArgs = @( + 'runtests', + '--testplan', $planFile, + '--project', $ProjectPath, + '--company', $Company, + '--server', $Connection.Server, + '--serverinstance', $Connection.ServerInstance, + '--port', "$($Connection.Port)", + '--environmenttype', 'OnPrem', + '--authentication', 'UserPassword', + '--tenant', $Tenant, + '--raw' + ) + + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $output = & al @alArgs 2>&1 + $sw.Stop() + + $lines = @($output | ForEach-Object { "$_" }) + $connected = ($lines | Where-Object { $_ -match 'Test run completed:' }).Count -gt 0 + $parsed = ConvertFrom-AlBatchOutput -OutputLines $lines + + if ($connected -and $parsed.Count -eq 0) { + Write-Host "::warning::batched al runtests connected but produced no parseable per-codeunit results. Raw output follows:" + Write-Host ($lines -join "`n") + } + + return @{ + Results = $parsed + ElapsedSec = [Math]::Round($sw.Elapsed.TotalSeconds, 3) + Raw = ($lines -join "`n") + Connected = $connected + } + } + finally { + Remove-Item $planFile -Force -ErrorAction SilentlyContinue + } +} + +<# +.SYNOPSIS + Appends a JUnit for one codeunit to the given document, matching the + exact schema BcContainerHelper produces. +.PARAMETER Doc + The JUnit XmlDocument being built. +.PARAMETER TestSuitesNode + The root element to append to. +.PARAMETER Codeunit + The codeunit object (.Id, .Name). +.PARAMETER RequestedMethods + The method names that were requested for this codeunit. +.PARAMETER MethodResults + The parsed per-method result map for this codeunit. +.PARAMETER ExtensionId + The extension (app) id. +.PARAMETER AppName + The app name. +.PARAMETER Hostname + The runner host name. +.PARAMETER ElapsedSec + The elapsed time (seconds) attributed to this codeunit. +.OUTPUTS + [int] Number of failing methods in this codeunit. +#> +function Add-JUnitTestSuite { + param( + [Parameter(Mandatory = $true)][System.Xml.XmlDocument] $Doc, + [Parameter(Mandatory = $true)][System.Xml.XmlElement] $TestSuitesNode, + [Parameter(Mandatory = $true)] $Codeunit, + [Parameter(Mandatory = $true)][string[]] $RequestedMethods, + [Parameter(Mandatory = $true)][hashtable] $MethodResults, + [Parameter(Mandatory = $true)][string] $ExtensionId, + [Parameter(Mandatory = $true)][string] $AppName, + [Parameter(Mandatory = $true)][string] $Hostname, + [Parameter(Mandatory = $true)][double] $ElapsedSec + ) + + $ci = [System.Globalization.CultureInfo]::InvariantCulture + $suiteName = "$($Codeunit.Id) $($Codeunit.Name)" + + $suite = $Doc.CreateElement("testsuite") + $suite.SetAttribute("name", $suiteName) + $suite.SetAttribute("timestamp", (Get-Date -Format s)) + $suite.SetAttribute("hostname", $Hostname) + + $props = $Doc.CreateElement("properties") + $suite.AppendChild($props) | Out-Null + $extProp = $Doc.CreateElement("property") + $extProp.SetAttribute("name", "extensionid") + $extProp.SetAttribute("value", $ExtensionId) + $props.AppendChild($extProp) | Out-Null + if ($AppName) { + $appProp = $Doc.CreateElement("property") + $appProp.SetAttribute("name", "appName") + $appProp.SetAttribute("value", $AppName) + $props.AppendChild($appProp) | Out-Null + } + + $failed = 0 + $skipped = 0 + foreach ($method in $RequestedMethods) { + $res = $MethodResults[$method] + + $tc = $Doc.CreateElement("testcase") + $tc.SetAttribute("classname", $suiteName) + $tc.SetAttribute("name", $method) + + if ($null -eq $res) { + # Method was requested but the runner produced no result (e.g. connect failure) -> error. + $tc.SetAttribute("time", "0") + $failure = $Doc.CreateElement("failure") + $failure.SetAttribute("message", "No result produced by al runtests") + $failure.InnerText = "" + $tc.AppendChild($failure) | Out-Null + $failed++ + } + else { + $tc.SetAttribute("time", ([Math]::Round($res.Ms / 1000.0, 3)).ToString($ci)) + switch ($res.Outcome) { + 'Fail' { + $failure = $Doc.CreateElement("failure") + $failure.SetAttribute("message", "$($res.Message)") + $failure.InnerText = "$($res.Stacktrace)".Replace(";", "`n") + $tc.AppendChild($failure) | Out-Null + $failed++ + } + 'Skip' { + $sk = $Doc.CreateElement("skipped") + $tc.AppendChild($sk) | Out-Null + $skipped++ + } + } + } + $suite.AppendChild($tc) | Out-Null + } + + $suite.SetAttribute("tests", "$($RequestedMethods.Count)") + $suite.SetAttribute("errors", "0") + $suite.SetAttribute("failures", "$failed") + $suite.SetAttribute("skipped", "$skipped") + $suite.SetAttribute("time", ([Math]::Round($ElapsedSec, 3)).ToString($ci)) + + $TestSuitesNode.AppendChild($suite) | Out-Null + return $failed +} + +<# +.SYNOPSIS + Runs all of a single app's test codeunits through `al runtests` and writes a JUnit results file. +.DESCRIPTION + The built-in default test runner for the RunTests action. Expects $Parameters to contain + containerName, credential, extensionId and (optionally) tenant, companyName, appName, + disabledTests and JUnitResultFileName. Runs the app's codeunits batched in one `--testplan` + invocation, then re-runs any codeunit that produced no result or a failure once in isolation, + and appends a BcContainerHelper-schema JUnit result. Returns whether every executed method + passed. +.PARAMETER Parameters + The BcContainerHelper-shaped test parameters built by Invoke-AlGoTestRun. +.OUTPUTS + [bool] $true if all methods passed and every codeunit connected; $false otherwise. +#> +function Invoke-AlToolTestRun { + param( + [Parameter(Mandatory = $true)][hashtable] $Parameters + ) + + Install-AlTool | Out-Null + + $containerName = $Parameters.containerName + $tenant = if ($Parameters.ContainsKey("tenant") -and $Parameters.tenant) { "$($Parameters.tenant)" } else { "default" } + $extensionId = "$($Parameters.extensionId)" + $appName = if ($Parameters.ContainsKey("appName")) { "$($Parameters.appName)" } else { "" } + $companyName = if ($Parameters.ContainsKey("companyName")) { "$($Parameters.companyName)" } else { "" } + + if ([string]::IsNullOrWhiteSpace($extensionId)) { + throw "Invoke-AlToolTestRun requires 'extensionId' in parameters." + } + + # Expose credentials to the al CLI (only auth channel it supports for on-prem UserPassword). + # Set inside the try below so the finally always clears them from the process environment. + if ($Parameters.credential -isnot [System.Management.Automation.PSCredential]) { + throw "Invoke-AlToolTestRun requires a PSCredential in parameters.credential." + } + + try { + $env:BC_SERVER_USERNAME = $Parameters.credential.UserName + $env:BC_SERVER_PASSWORD = $Parameters.credential.GetNetworkCredential().Password + + $connection = Get-AlToolConnection -ContainerName $containerName + $projectPath = New-AlToolProject -ContainerName $containerName -Tenant $tenant -Connection $connection + $company = Get-AlToolCompany -ContainerName $containerName -Tenant $tenant -CompanyName $companyName + if ([string]::IsNullOrWhiteSpace($company)) { + throw "Could not resolve a company to run tests against in container '$containerName'." + } + + Write-Host "altool run: app='$appName' extensionId=$extensionId company='$company' server='$($connection.Server)' instance='$($connection.ServerInstance)' port=$($connection.Port) tenant='$tenant'" + + $codeunits = @(Get-AlToolTestCodeunits -Parameters $Parameters) + Write-Host "Enumerated $($codeunits.Count) test codeunit(s) for app '$appName'." + if ($codeunits.Count -eq 0) { + Write-Host "No test codeunits to run for app '$appName'; nothing to do." + return $true + } + + $hostname = [System.Net.Dns]::GetHostName() + + # Append to an existing JUnit file when present (Invoke-AlGoTestRun runs one app at a time + # into the same TestResults.xml), matching BCH's AppendToJUnitResultFile behavior. + $junitFile = if ($Parameters.ContainsKey("JUnitResultFileName")) { $Parameters.JUnitResultFileName } else { "" } + $doc = New-Object System.Xml.XmlDocument + $suites = $null + if (-not [string]::IsNullOrWhiteSpace($junitFile) -and (Test-Path $junitFile)) { + try { + $doc.Load($junitFile) + $suites = $doc.DocumentElement + if (-not $suites -or $suites.LocalName -ne 'testsuites') { $suites = $null; $doc = New-Object System.Xml.XmlDocument } + } + catch { + Write-Host "WARNING: Could not load existing JUnit file '$junitFile' ($($_.Exception.Message)); starting fresh." + $doc = New-Object System.Xml.XmlDocument + $suites = $null + } + } + if (-not $suites) { + $doc.AppendChild($doc.CreateXmlDeclaration("1.0", "UTF-8", $null)) | Out-Null + $suites = $doc.CreateElement("testsuites") + $doc.AppendChild($suites) | Out-Null + } + + $allPassed = $true + $merged = @{} # merged[codeunitId] = @{ method -> result } + $totalElapsed = 0.0 + + # BATCHED execution: run every codeunit for this app in a SINGLE `al runtests --testplan` + # invocation (one auth + one hub connection + one server Initialize). + $groups = @($codeunits | ForEach-Object { @{ Id = "$($_.Id)"; Methods = @($_.Tests | ForEach-Object { "$_" }) } }) + + $batch = Invoke-AlBatchRunTests -Groups $groups -ProjectPath $projectPath -Company $company ` + -Tenant $tenant -Connection $connection + if (-not $batch.Connected) { + Write-Host "::warning::batched al runtests did not complete for app '$appName'. Raw output:" + Write-Host $batch.Raw + $allPassed = $false + } + foreach ($k in $batch.Results.Keys) { $merged[$k] = $batch.Results[$k] } + $totalElapsed = [double]$batch.ElapsedSec + + # Isolated fallback + rerun pass. Each affected codeunit runs in its OWN `al runtests` call + # (a fresh session): a no-result codeunit (a state-sensitive codeunit can produce no results + # mid-batch) is retried for correctness, and a failed codeunit is retried once to recover + # flaky failures (mirroring BCH's rerun of failed tests). + $isoGroups = @() + foreach ($cu in $codeunits) { + $cid = "$($cu.Id)" + $requested = @($cu.Tests | ForEach-Object { "$_" }) + $cuResults = $merged[$cid] + $retryMethods = @($requested | Where-Object { + $r = if ($cuResults) { $cuResults[$_] } else { $null } + ($null -eq $r) -or ($r.Outcome -eq 'Fail') + }) + if ($retryMethods.Count -gt 0) { + $isoGroups += @{ Id = $cid; Methods = $retryMethods; Name = $cu.Name } + } + } + if ($isoGroups.Count -gt 0) { + Write-Host ("isolated fallback/rerun: {0} codeunit(s) (each in its own session)" -f $isoGroups.Count) + foreach ($g in $isoGroups) { + $iso = Invoke-AlRunTestsForCodeunit -CodeunitId $g.Id -Methods $g.Methods ` + -ProjectPath $projectPath -Company $company -Tenant $tenant -Connection $connection + $totalElapsed += [double]$iso.ElapsedSec + if (-not $merged.ContainsKey($g.Id)) { $merged[$g.Id] = @{} } + foreach ($mName in $iso.Results.Keys) { $merged[$g.Id][$mName] = $iso.Results[$mName] } + } + } + + # Distribute the app's REAL al wall-clock across its codeunits weighted by each codeunit's + # method-ms share (al's per-method `ms` under-reports real work, so it is used only for + # weighting, not as the absolute suite time). Equal split as a fallback. + $cuMsShare = @{} + $grandMs = 0.0 + foreach ($cu in $codeunits) { + $cuResults = $merged["$($cu.Id)"] + $ms = 0.0 + if ($cuResults) { foreach ($mName in $cuResults.Keys) { $ms += [double]$cuResults[$mName].Ms } } + $cuMsShare["$($cu.Id)"] = $ms + $grandMs += $ms + } + + # Build one JUnit per codeunit from the merged results. + $idx = 0 + foreach ($cu in $codeunits) { + $idx++ + $methods = @($cu.Tests | ForEach-Object { "$_" }) + $cuResults = $merged["$($cu.Id)"] + if ($null -eq $cuResults) { $cuResults = @{} } + + if ($grandMs -gt 0) { + $suiteSec = $totalElapsed * ($cuMsShare["$($cu.Id)"] / $grandMs) + } + elseif ($codeunits.Count -gt 0) { + $suiteSec = $totalElapsed / $codeunits.Count + } + else { + $suiteSec = 0.0 + } + + $failed = Add-JUnitTestSuite -Doc $doc -TestSuitesNode $suites -Codeunit $cu ` + -RequestedMethods $methods -MethodResults $cuResults -ExtensionId $extensionId ` + -AppName $appName -Hostname $hostname -ElapsedSec $suiteSec + + if ($failed -gt 0) { $allPassed = $false } + + Write-Host ("[{0}/{1}] cu {2} '{3}' -> {4} failed of {5} method(s)" -f ` + $idx, $codeunits.Count, $cu.Id, $cu.Name, $failed, $methods.Count) + } + Write-Host ("Run for app '{0}': {1} codeunit(s) in {2}s real al wall-clock." -f ` + $appName, $codeunits.Count, [Math]::Round($totalElapsed, 2)) + + if (-not [string]::IsNullOrWhiteSpace($junitFile)) { + $dir = [System.IO.Path]::GetDirectoryName($junitFile) + if ($dir -and -not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null } + $doc.Save($junitFile) + Write-Host "Wrote JUnit results for app '$appName' to $junitFile" + } + else { + Write-Host "WARNING: No JUnitResultFileName in parameters; results not persisted for app '$appName'." + } + + return $allPassed + } + finally { + # Do not let the container credential linger in the process environment after the run. + Remove-Item Env:\BC_SERVER_USERNAME -ErrorAction SilentlyContinue + Remove-Item Env:\BC_SERVER_PASSWORD -ErrorAction SilentlyContinue + } +} + +Export-ModuleMember -Function Install-AlTool, Get-AlToolConnection, New-AlToolProject, Get-AlToolCompany, ` + Get-DisabledTestKeySet, Get-AlToolTestCodeunits, ConvertFrom-AlRunTestsOutput, ConvertFrom-AlBatchOutput, ` + ConvertTo-AlTestPlanJson, Invoke-AlRunTestsForCodeunit, Invoke-AlBatchRunTests, ` + Add-JUnitTestSuite, Invoke-AlToolTestRun diff --git a/Actions/RunTests/README.md b/Actions/RunTests/README.md index 12a4aa9877..9bebfd64ac 100644 --- a/Actions/RunTests/README.md +++ b/Actions/RunTests/README.md @@ -6,6 +6,16 @@ This action only does anything when the `useSeparateTestAction` setting is enabl Only normal tests (testFolders) are handled here. BCPT and page scripting tests continue to be executed by the RunPipeline action. +## Test runner + +By default this action runs the tests through Microsoft's headless `al runtests` (AlTool) runner. It resolves the kept-alive container's connection settings (server, instance and developer-services port) from the container name, installs the AL developer tools as a `dotnet` global tool (`dotnet tool install Microsoft.Dynamics.BusinessCentral.Development.Tools --prerelease`), enumerates the test codeunits and runs them against the container, then writes the same `TestResults.xml` (JUnit) schema BcContainerHelper produces so downstream test result analysis is unchanged. + +To run the tests through BcContainerHelper (or any other runner) instead, add a `RunTestsInBcContainer` override script under the project's `.AL-Go` folder. When present, it fully replaces the built-in AlTool runner and is called once per test app with the same parameters BcContainerHelper's `Run-TestsInBcContainer` expects. + +### Known limitations + +The built-in AlTool runner does not run `Legacy` test-type codeunits or tests that require UI or client-callback interaction. Repositories that rely on those should run their tests through BcContainerHelper by supplying a `RunTestsInBcContainer` override script (as described above), which fully replaces the AlTool runner. + ## INPUT ### ENV variables diff --git a/Actions/RunTests/RunTests.ps1 b/Actions/RunTests/RunTests.ps1 index 2d0f86368b..e5acf2c7d9 100644 --- a/Actions/RunTests/RunTests.ps1 +++ b/Actions/RunTests/RunTests.ps1 @@ -80,7 +80,7 @@ if (-not $containerName) { # Credentials used to connect to the build container. $credential = Get-TestRunnerCredential -# A RunTestsInBcContainer override script, if present, replaces the built-in BcContainerHelper test runner. +# A RunTestsInBcContainer override script, if present, replaces the built-in AlTool test runner. $overrideParams = Get-ScriptOverrides -ALGoFolderName (Join-Path $projectPath ".AL-Go") -OverrideScriptNames @("RunTestsInBcContainer") Invoke-AlGoTestRun ` diff --git a/Actions/RunTests/RunTests.psm1 b/Actions/RunTests/RunTests.psm1 index b06d01e3d1..925e1994bc 100644 --- a/Actions/RunTests/RunTests.psm1 +++ b/Actions/RunTests/RunTests.psm1 @@ -5,8 +5,14 @@ Contains the logic for running the normal tests (testFolders) of an AL-Go project against a build container that was created and kept alive by the RunPipeline action. Kept in a module so the logic can be unit tested independently of the action entry script. + + By default the tests are run through the AlTool (`al runtests`) runner in AlToolTestRunner.psm1. + A RunTestsInBcContainer override script, when supplied, replaces that default with the + BcContainerHelper test runner (this is how, for example, BCApps supplies its own runner). #> +Import-Module (Join-Path $PSScriptRoot 'AlToolTestRunner.psm1' -Resolve) -DisableNameChecking -Force + function Get-TestAppsToRun { <# .SYNOPSIS @@ -55,9 +61,9 @@ function Invoke-AlGoTestRun { Runs the normal tests for an AL-Go project against a kept-alive build container. .DESCRIPTION Runs tests in each test app against the given container and writes the results to - testResultsFile in JUnit format. Honors the treatTestFailuresAsWarnings setting. When a - RunTestsInBcContainer override script is provided, it is used instead of the built-in - BcContainerHelper test runner. + testResultsFile in JUnit format. Honors the treatTestFailuresAsWarnings setting. By default + the tests are run through the AlTool (`al runtests`) runner. When a RunTestsInBcContainer + override script is provided, it is used instead of the built-in AlTool runner. .PARAMETER settings The (analyzed) AL-Go settings hashtable. .PARAMETER projectPath @@ -69,7 +75,7 @@ function Invoke-AlGoTestRun { .PARAMETER installTestAppsJson Path to a JSON file with the list of installed test apps. .PARAMETER runTestsOverride - Optional scriptblock overriding the BcContainerHelper test runner (RunTestsInBcContainer). + Optional scriptblock overriding the built-in AlTool test runner (RunTestsInBcContainer). #> Param( [hashtable] $settings, @@ -120,7 +126,7 @@ function Invoke-AlGoTestRun { $passed = & $runTestsOverride -parameters $runTestsParams } else { - $passed = Run-TestsInBcContainer @runTestsParams + $passed = Invoke-AlToolTestRun -Parameters $runTestsParams } if (-not $passed) { diff --git a/RELEASENOTES.md b/RELEASENOTES.md index f1659d6ab0..cc6c42e8a4 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -2,6 +2,11 @@ A new `useSeparateTestAction` setting (default `false`) lets you move normal test execution (`testFolders`) out of the `RunPipeline` action and into a new dedicated `RunTests` action. When enabled, `RunPipeline` still compiles, publishes and installs the apps and keeps the build container alive, but does not run the normal tests. A new `RunTests` action then runs the tests against that same container and produces the same `TestResults.xml`. Only normal tests are affected; BCPT and page scripting tests are still executed by `RunPipeline`. When the setting is `false`, behavior is unchanged. +By default, the `RunTests` action runs the tests through Microsoft's headless `al runtests` (AlTool) runner instead of BcContainerHelper. It resolves the kept-alive container's connection settings, installs the AL developer tools (`dotnet tool install --prerelease`) and emits the same `TestResults.xml` schema, so downstream test result analysis is unchanged. To run the tests through BcContainerHelper (or any other runner) instead, supply a `RunTestsInBcContainer` override script; it fully replaces the built-in AlTool runner. + +> [!NOTE] +> The built-in AlTool runner does not run `Legacy` test-type codeunits or tests that require UI or client-callback interaction. Projects that rely on those should supply a `RunTestsInBcContainer` override script to run their tests through BcContainerHelper instead. + ### New `doNotPerformUpgrade` setting AL-Go now supports a new `doNotPerformUpgrade` setting that is passed through to `Run-AlPipeline`. Use it to skip the upgrade phase while still running the rest of the pipeline. diff --git a/Tests/AlToolTestRunner.Test.ps1 b/Tests/AlToolTestRunner.Test.ps1 new file mode 100644 index 0000000000..693be26172 --- /dev/null +++ b/Tests/AlToolTestRunner.Test.ps1 @@ -0,0 +1,260 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'Mock/callback parameters must match function signatures')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'Test-only credential')] +param() + +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 + +Import-Module (Join-Path $PSScriptRoot '../Actions/RunTests/AlToolTestRunner.psm1' -Resolve) -DisableNameChecking -Force + +Describe 'AlToolTestRunner.psm1 Tests' { + + BeforeAll { + # Re-import in the run phase so the module functions are guaranteed to be available even when + # this file runs in the same Invoke-Pester session as RunTests.Test.ps1. RunTests.psm1 + # imports AlToolTestRunner.psm1 as a nested module with -Force, which removes the standalone + # module's functions from the global scope during discovery. + Import-Module (Join-Path $PSScriptRoot '../Actions/RunTests/AlToolTestRunner.psm1' -Resolve) -DisableNameChecking -Force + } + + Context 'ConvertFrom-AlRunTestsOutput' { + It 'Parses pass and fail results, dropping phantom OnRun and aggregate entries' { + $lines = @( + 'Test run completed: 3 tests, 1 failed', + 'Results:', + 'PASS OnRun (5ms)', + 'PASS MyPassingTest (12ms)', + 'FAIL MyFailingTest (8ms)', + ' Assert.AreEqual failed. Expected 1, got 2.', + 'AL Callstack:', + ' "My Codeunit"(CodeUnit 130001).MyFailingTest line 3', + 'PASS (25ms)' + ) + + $results = ConvertFrom-AlRunTestsOutput -OutputLines $lines + + $results.Keys.Count | Should -Be 2 + $results.ContainsKey('OnRun') | Should -BeFalse + $results['MyPassingTest'].Outcome | Should -Be 'Pass' + $results['MyPassingTest'].Ms | Should -Be 12 + $results['MyFailingTest'].Outcome | Should -Be 'Fail' + $results['MyFailingTest'].Message | Should -Match 'Assert.AreEqual failed' + $results['MyFailingTest'].Stacktrace | Should -Match 'MyFailingTest line 3' + } + + It 'Returns an empty map when there is no Results block' { + $results = ConvertFrom-AlRunTestsOutput -OutputLines @('Some unrelated output', 'No results here') + $results.Keys.Count | Should -Be 0 + } + + It 'Parses skipped results' { + $lines = @('Results:', 'SKIP MySkippedTest (0ms)') + $results = ConvertFrom-AlRunTestsOutput -OutputLines $lines + $results['MySkippedTest'].Outcome | Should -Be 'Skip' + } + } + + Context 'ConvertFrom-AlBatchOutput' { + It 'Splits batched output into per-codeunit result maps' { + $lines = @( + '===== Codeunit 130001 =====', + 'Results:', + 'PASS TestA (3ms)', + '===== Codeunit 130002 =====', + 'Results:', + 'FAIL TestB (7ms)', + ' boom', + 'AL Callstack:', + ' stack' + ) + + $byCodeunit = ConvertFrom-AlBatchOutput -OutputLines $lines + + $byCodeunit.Keys.Count | Should -Be 2 + $byCodeunit['130001']['TestA'].Outcome | Should -Be 'Pass' + $byCodeunit['130002']['TestB'].Outcome | Should -Be 'Fail' + } + } + + Context 'ConvertTo-AlTestPlanJson' { + It 'Emits arrays at both levels for a single codeunit with a single method' { + $json = ConvertTo-AlTestPlanJson -Groups @(@{ Id = '130001'; Methods = @('OnlyTest') }) + + $json | Should -Match '^\[\{' + $json | Should -Match '\}\]$' + $parsed = $json | ConvertFrom-Json + @($parsed).Count | Should -Be 1 + $parsed[0].codeunitId | Should -Be 130001 + @($parsed[0].testMethods).Count | Should -Be 1 + $parsed[0].testMethods[0] | Should -Be 'OnlyTest' + } + + It 'Serializes multiple codeunits and methods' { + $groups = @( + @{ Id = '1'; Methods = @('A', 'B') }, + @{ Id = '2'; Methods = @('C') } + ) + $parsed = (ConvertTo-AlTestPlanJson -Groups $groups) | ConvertFrom-Json + @($parsed).Count | Should -Be 2 + @($parsed[0].testMethods).Count | Should -Be 2 + $parsed[1].codeunitId | Should -Be 2 + } + } + + Context 'Get-DisabledTestKeySet' { + It 'Builds per-method keys and whole-codeunit sets from a wildcard' { + $disabled = @( + [PSCustomObject]@{ codeunitName = 'My Tests'; method = 'TestOne' }, + [PSCustomObject]@{ codeunitName = 'Whole CU'; method = '*' } + ) + + $lookup = Get-DisabledTestKeySet -DisabledTests $disabled + + $lookup.Methods.ContainsKey('my tests::testone') | Should -BeTrue + $lookup.Codeunits.ContainsKey('whole cu') | Should -BeTrue + $lookup.Methods.ContainsKey('whole cu::*') | Should -BeFalse + } + + It 'Returns empty sets for an empty list' { + $lookup = Get-DisabledTestKeySet -DisabledTests @() + $lookup.Methods.Count | Should -Be 0 + $lookup.Codeunits.Count | Should -Be 0 + } + } + + Context 'Get-AlToolTestCodeunits' { + It 'Enumerates codeunits and filters disabled methods and whole codeunits' { + Mock -ModuleName AlToolTestRunner Get-TestsFromBcContainer { + @( + [PSCustomObject]@{ Id = 130001; Name = 'My Tests'; Tests = @('TestOne', 'TestTwo') }, + [PSCustomObject]@{ Id = 130002; Name = 'Whole CU'; Tests = @('X', 'Y') } + ) + } + $params = @{ + containerName = 'test' + credential = (New-Object System.Management.Automation.PSCredential('admin', (ConvertTo-SecureString 'password' -AsPlainText -Force))) + extensionId = [Guid]::NewGuid().ToString() + disabledTests = @( + [PSCustomObject]@{ codeunitName = 'My Tests'; method = 'TestTwo' }, + [PSCustomObject]@{ codeunitName = 'Whole CU'; method = '*' } + ) + } + + $codeunits = @(Get-AlToolTestCodeunits -Parameters $params) + + $codeunits.Count | Should -Be 1 + $codeunits[0].Name | Should -Be 'My Tests' + @($codeunits[0].Tests).Count | Should -Be 1 + $codeunits[0].Tests[0] | Should -Be 'TestOne' + } + + It 'Returns every codeunit when there are no disabled tests' { + Mock -ModuleName AlToolTestRunner Get-TestsFromBcContainer { + @([PSCustomObject]@{ Id = 130001; Name = 'My Tests'; Tests = @('TestOne') }) + } + $params = @{ + containerName = 'test' + credential = (New-Object System.Management.Automation.PSCredential('admin', (ConvertTo-SecureString 'password' -AsPlainText -Force))) + extensionId = [Guid]::NewGuid().ToString() + } + + $codeunits = @(Get-AlToolTestCodeunits -Parameters $params) + $codeunits.Count | Should -Be 1 + } + } + + Context 'Get-AlToolConnection' { + It 'Reads server instance and developer services port from the container configuration' { + Mock -ModuleName AlToolTestRunner Get-BcContainerServerConfiguration { + [PSCustomObject]@{ ServerInstance = 'MyBC'; DeveloperServicesPort = 7145 } + } + + $connection = Get-AlToolConnection -ContainerName 'mycontainer' + + $connection.Server | Should -Be 'http://mycontainer' + $connection.ServerInstance | Should -Be 'MyBC' + $connection.Port | Should -Be 7145 + } + + It 'Falls back to conventional defaults when configuration cannot be read' { + Mock -ModuleName AlToolTestRunner Get-BcContainerServerConfiguration { throw 'no such container' } + + $connection = Get-AlToolConnection -ContainerName 'mycontainer' + + $connection.Server | Should -Be 'http://mycontainer' + $connection.ServerInstance | Should -Be 'BC' + $connection.Port | Should -Be 7049 + } + } + + Context 'Get-AlToolCompany' { + It 'Honors an explicitly requested company name without querying the container' { + Mock -ModuleName AlToolTestRunner Get-CompanyInBcContainer { throw 'should not be called' } + $company = Get-AlToolCompany -ContainerName 'test' -Tenant 'default' -CompanyName 'CRONUS' + $company | Should -Be 'CRONUS' + } + + It 'Falls back to the container default company, preferring an evaluation company' { + Mock -ModuleName AlToolTestRunner Get-CompanyInBcContainer { + @( + [PSCustomObject]@{ companyName = 'CRONUS Real'; evaluationCompany = $false }, + [PSCustomObject]@{ companyName = 'CRONUS Eval'; evaluationCompany = $true } + ) + } + $company = Get-AlToolCompany -ContainerName 'test' -Tenant 'default' + $company | Should -Be 'CRONUS Eval' + } + } + + Context 'Add-JUnitTestSuite' { + BeforeEach { + $script:doc = New-Object System.Xml.XmlDocument + $script:doc.AppendChild($script:doc.CreateXmlDeclaration("1.0", "UTF-8", $null)) | Out-Null + $script:suites = $script:doc.CreateElement("testsuites") + $script:doc.AppendChild($script:suites) | Out-Null + } + + It 'Writes a passing suite with correct counts and no failure nodes' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + $methodResults = @{ TestA = @{ Outcome = 'Pass'; Ms = 10; Message = ''; Stacktrace = '' } } + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('TestA') -MethodResults $methodResults -ExtensionId 'ext-id' -AppName 'MyApp' ` + -Hostname 'host' -ElapsedSec 1.5 + + $failed | Should -Be 0 + $suite = $script:suites.SelectSingleNode('testsuite') + $suite.GetAttribute('name') | Should -Be '130001 My Tests' + $suite.GetAttribute('tests') | Should -Be '1' + $suite.GetAttribute('failures') | Should -Be '0' + $suite.SelectNodes('testcase/failure').Count | Should -Be 0 + } + + It 'Marks a requested method with no result as a failure' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('MissingTest') -MethodResults @{} -ExtensionId 'ext-id' -AppName 'MyApp' ` + -Hostname 'host' -ElapsedSec 0.0 + + $failed | Should -Be 1 + $suite = $script:suites.SelectSingleNode('testsuite') + $suite.GetAttribute('failures') | Should -Be '1' + $suite.SelectSingleNode('testcase/failure').GetAttribute('message') | Should -Match 'No result produced' + } + + It 'Records a failing method with its message and stacktrace' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + $methodResults = @{ TestA = @{ Outcome = 'Fail'; Ms = 4; Message = 'boom'; Stacktrace = 'line1;line2' } } + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('TestA') -MethodResults $methodResults -ExtensionId 'ext-id' -AppName 'MyApp' ` + -Hostname 'host' -ElapsedSec 0.5 + + $failed | Should -Be 1 + $failureNode = $script:suites.SelectSingleNode('testsuite/testcase/failure') + $failureNode.GetAttribute('message') | Should -Be 'boom' + $failureNode.InnerText | Should -Match 'line1' + $failureNode.InnerText | Should -Match 'line2' + } + } +} diff --git a/Tests/RunTests.Test.ps1 b/Tests/RunTests.Test.ps1 index 89409e94df..8b72b31c86 100644 --- a/Tests/RunTests.Test.ps1 +++ b/Tests/RunTests.Test.ps1 @@ -182,4 +182,54 @@ Describe 'RunTests.psm1 Tests' { Remove-Item -Path $projectPath -Recurse -Force } } + + Context 'Invoke-AlGoTestRun (default AlTool runner)' { + It 'Runs the AlTool runner for every test app when no override is supplied' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + Mock -ModuleName RunTests Invoke-AlToolTestRun { return $true } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app', 'App2.Test.app') + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential } | Should -Not -Throw + + Should -Invoke -ModuleName RunTests Invoke-AlToolTestRun -Times 2 -Exactly + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Passes the container and credential through to the AlTool runner' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + Mock -ModuleName RunTests Invoke-AlToolTestRun { return $true } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = 'CRONUS'; treatTestFailuresAsWarnings = $false } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'mycontainer' -credential $testCredential + + Should -Invoke -ModuleName RunTests Invoke-AlToolTestRun -Times 1 -Exactly -ParameterFilter { + $Parameters.containerName -eq 'mycontainer' -and $Parameters.companyName -eq 'CRONUS' -and ($Parameters.credential -is [System.Management.Automation.PSCredential]) + } + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Throws when the AlTool runner reports failure and treatTestFailuresAsWarnings is not set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + Mock -ModuleName RunTests Invoke-AlToolTestRun { return $false } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential } | Should -Throw + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Does not throw when the AlTool runner reports failure but treatTestFailuresAsWarnings is set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + Mock -ModuleName RunTests Invoke-AlToolTestRun { return $false } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $true } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential } | Should -Not -Throw + + Remove-Item -Path $projectPath -Recurse -Force + } + } } From 3ab5c81ba6f7d6e4bc01ae43b40ac31a0abd3343 Mon Sep 17 00:00:00 2001 From: spetersenms Date: Mon, 10 Aug 2026 08:53:13 +0200 Subject: [PATCH 5/6] Remove unsupported altool --testplan batching from RunTests The official released al runtests tool does not support --testplan, so the batch-first path in AlToolTestRunner hard-set llPassed=false whenever the batch failed to connect and never reset it, making every run falsely report FAILED. Remove the batching path entirely and run each test codeunit in its own l runtests --testmethods --raw invocation, with the existing rerun-on-failure pass. llPassed now derives solely from real per-method outcomes via Add-JUnitTestSuite. - Delete Invoke-AlBatchRunTests, ConvertFrom-AlBatchOutput, ConvertTo-AlTestPlanJson and their Export-ModuleMember entries. - Update README and RELEASENOTES to per-codeunit wording. - Drop the two batch-specific unit test contexts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Actions/RunTests/AlToolTestRunner.psm1 | 196 ++++--------------------- Actions/RunTests/README.md | 2 +- RELEASENOTES.md | 2 +- Tests/AlToolTestRunner.Test.ps1 | 47 ------ 4 files changed, 28 insertions(+), 219 deletions(-) diff --git a/Actions/RunTests/AlToolTestRunner.psm1 b/Actions/RunTests/AlToolTestRunner.psm1 index 910492b1f4..b43636dbbd 100644 --- a/Actions/RunTests/AlToolTestRunner.psm1 +++ b/Actions/RunTests/AlToolTestRunner.psm1 @@ -13,9 +13,8 @@ host-side from the container name via BcContainerHelper, and generates a throw-away AL project with a launch.json so `al runtests` has connection settings. 3. Enumerates the app's test codeunits + methods via Get-TestsFromBcContainer. - 4. Runs `al runtests` batched in one `--testplan` invocation (single auth/connect), with an - isolated per-codeunit fallback/rerun pass for any codeunit that produced no result or a - failure. + 4. Runs `al runtests --testmethods` once per test codeunit (each in its own + session), with a rerun pass that retries any method that produced no result or a failure. 5. Emits a JUnit results file matching the exact schema BcContainerHelper produces, so the downstream AnalyzeTests step keeps working unchanged. @@ -409,80 +408,6 @@ function ConvertFrom-AlRunTestsOutput { return $results } -<# -.SYNOPSIS - Splits batched `al runtests --testplan --raw` output into per-codeunit result maps. -.DESCRIPTION - The batched runner emits one block per codeunit, each preceded by a "===== Codeunit =====" - marker and containing a "Results:" section. This splits on the marker and parses each block with - ConvertFrom-AlRunTestsOutput, keyed by codeunit id. -.PARAMETER OutputLines - The lines of batched `al runtests --raw` output. -.OUTPUTS - [hashtable] "" -> (method-name -> @{ Outcome; Ms; Message; Stacktrace }) -#> -function ConvertFrom-AlBatchOutput { - param( - [Parameter(Mandatory = $true)][AllowEmptyCollection()][AllowEmptyString()][string[]] $OutputLines - ) - - $byCodeunit = @{} - $markerRegex = '^\s*=====\s*Codeunit\s+(\d+)\s*=====\s*$' - - $currentId = $null - $currentLines = New-Object System.Collections.Generic.List[string] - - $flush = { - if ($null -ne $currentId) { - $byCodeunit["$currentId"] = ConvertFrom-AlRunTestsOutput -OutputLines @($currentLines.ToArray()) - } - } - - foreach ($line in $OutputLines) { - $m = [regex]::Match($line, $markerRegex) - if ($m.Success) { - & $flush - $currentId = $m.Groups[1].Value - $currentLines = New-Object System.Collections.Generic.List[string] - continue - } - if ($null -ne $currentId) { $currentLines.Add($line) } - } - & $flush - - return $byCodeunit -} - -<# -.SYNOPSIS - Serializes a batch of test groups into the JSON array the al `--testplan` option expects. -.DESCRIPTION - Built by hand (not ConvertTo-Json) because PowerShell's ConvertTo-Json collapses single-element - arrays to a scalar/object - a one-codeunit plan would become {..} not [{..}], and a one-method - list "M" not ["M"] - which the al tool rejects. This guarantees arrays at both levels for any count. -.PARAMETER Groups - Array of @{ Id; Methods } describing the codeunits (and enabled methods) to run. -.OUTPUTS - [string] JSON array: [{ "codeunitId": N, "testMethods": [ ... ] }, ...] -#> -function ConvertTo-AlTestPlanJson { - param( - [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]] $Groups - ) - - $sb = New-Object System.Text.StringBuilder - [void]$sb.Append('[') - $firstGroup = $true - foreach ($g in $Groups) { - if (-not $firstGroup) { [void]$sb.Append(',') } - $firstGroup = $false - $methodsJson = @($g.Methods | ForEach-Object { "$_" | ConvertTo-Json -Compress }) - [void]$sb.Append(('{{"codeunitId":{0},"testMethods":[{1}]}}' -f [int]$g.Id, ($methodsJson -join ','))) - } - [void]$sb.Append(']') - return $sb.ToString() -} - <# .SYNOPSIS Runs `al runtests` for one codeunit and returns the parsed per-method results plus raw output. @@ -549,76 +474,6 @@ function Invoke-AlRunTestsForCodeunit { } } -<# -.SYNOPSIS - Runs a batch of codeunits in ONE `al runtests --testplan` invocation (single connection + auth + - session) and returns per-codeunit results. This removes the per-codeunit connect/auth tax. -.PARAMETER Groups - Array of @{ Id; Methods } describing the codeunits (and enabled methods) to run. -.PARAMETER ProjectPath - The throw-away AL project folder. -.PARAMETER Company - The company to run against. -.PARAMETER Tenant - The tenant to connect to. -.PARAMETER Connection - The connection hashtable produced by Get-AlToolConnection. -.OUTPUTS - [hashtable] @{ Results (""->method map); ElapsedSec; Connected; Raw } -#> -function Invoke-AlBatchRunTests { - param( - [Parameter(Mandatory = $true)][object[]] $Groups, - [Parameter(Mandatory = $true)][string] $ProjectPath, - [Parameter(Mandatory = $true)][string] $Company, - [Parameter(Mandatory = $true)][string] $Tenant, - [Parameter(Mandatory = $true)][hashtable] $Connection - ) - - # Write the test plan as a JSON file (avoids command-line length limits with many codeunits). - $planFile = Join-Path ([System.IO.Path]::GetTempPath()) ("altool-plan-" + [System.Guid]::NewGuid().ToString('N') + ".json") - Set-Content -Path $planFile -Value (ConvertTo-AlTestPlanJson -Groups $Groups) -Encoding UTF8 - - try { - $alArgs = @( - 'runtests', - '--testplan', $planFile, - '--project', $ProjectPath, - '--company', $Company, - '--server', $Connection.Server, - '--serverinstance', $Connection.ServerInstance, - '--port', "$($Connection.Port)", - '--environmenttype', 'OnPrem', - '--authentication', 'UserPassword', - '--tenant', $Tenant, - '--raw' - ) - - $sw = [System.Diagnostics.Stopwatch]::StartNew() - $output = & al @alArgs 2>&1 - $sw.Stop() - - $lines = @($output | ForEach-Object { "$_" }) - $connected = ($lines | Where-Object { $_ -match 'Test run completed:' }).Count -gt 0 - $parsed = ConvertFrom-AlBatchOutput -OutputLines $lines - - if ($connected -and $parsed.Count -eq 0) { - Write-Host "::warning::batched al runtests connected but produced no parseable per-codeunit results. Raw output follows:" - Write-Host ($lines -join "`n") - } - - return @{ - Results = $parsed - ElapsedSec = [Math]::Round($sw.Elapsed.TotalSeconds, 3) - Raw = ($lines -join "`n") - Connected = $connected - } - } - finally { - Remove-Item $planFile -Force -ErrorAction SilentlyContinue - } -} - <# .SYNOPSIS Appends a JUnit for one codeunit to the given document, matching the @@ -732,14 +587,14 @@ function Add-JUnitTestSuite { .DESCRIPTION The built-in default test runner for the RunTests action. Expects $Parameters to contain containerName, credential, extensionId and (optionally) tenant, companyName, appName, - disabledTests and JUnitResultFileName. Runs the app's codeunits batched in one `--testplan` - invocation, then re-runs any codeunit that produced no result or a failure once in isolation, + disabledTests and JUnitResultFileName. Runs each of the app's test codeunits in its own + `al runtests` invocation, then re-runs any method that produced no result or a failure once, and appends a BcContainerHelper-schema JUnit result. Returns whether every executed method passed. .PARAMETER Parameters The BcContainerHelper-shaped test parameters built by Invoke-AlGoTestRun. .OUTPUTS - [bool] $true if all methods passed and every codeunit connected; $false otherwise. + [bool] $true if all executed methods passed; $false otherwise. #> function Invoke-AlToolTestRun { param( @@ -813,24 +668,26 @@ function Invoke-AlToolTestRun { $merged = @{} # merged[codeunitId] = @{ method -> result } $totalElapsed = 0.0 - # BATCHED execution: run every codeunit for this app in a SINGLE `al runtests --testplan` - # invocation (one auth + one hub connection + one server Initialize). - $groups = @($codeunits | ForEach-Object { @{ Id = "$($_.Id)"; Methods = @($_.Tests | ForEach-Object { "$_" }) } }) - - $batch = Invoke-AlBatchRunTests -Groups $groups -ProjectPath $projectPath -Company $company ` - -Tenant $tenant -Connection $connection - if (-not $batch.Connected) { - Write-Host "::warning::batched al runtests did not complete for app '$appName'. Raw output:" - Write-Host $batch.Raw - $allPassed = $false + # PRIMARY execution: run each test codeunit in its OWN `al runtests --testmethods` + # invocation (a fresh session per codeunit). The official altool has no batch/plan mode, + # so per-codeunit is the only supported execution model. + foreach ($cu in $codeunits) { + $cid = "$($cu.Id)" + $methods = @($cu.Tests | ForEach-Object { "$_" }) + $run = Invoke-AlRunTestsForCodeunit -CodeunitId $cid -Methods $methods ` + -ProjectPath $projectPath -Company $company -Tenant $tenant -Connection $connection + $totalElapsed += [double]$run.ElapsedSec + if (-not $run.Connected) { + Write-Host "::warning::al runtests did not complete for codeunit $cid ('$($cu.Name)') in app '$appName'. Raw output:" + Write-Host $run.Raw + } + $merged[$cid] = $run.Results } - foreach ($k in $batch.Results.Keys) { $merged[$k] = $batch.Results[$k] } - $totalElapsed = [double]$batch.ElapsedSec - # Isolated fallback + rerun pass. Each affected codeunit runs in its OWN `al runtests` call - # (a fresh session): a no-result codeunit (a state-sensitive codeunit can produce no results - # mid-batch) is retried for correctness, and a failed codeunit is retried once to recover - # flaky failures (mirroring BCH's rerun of failed tests). + # Rerun-on-failure pass. Each affected codeunit runs again in its OWN `al runtests` call + # (a fresh session): a method that produced no result (a state-sensitive codeunit can leave + # a method unreported) is retried for correctness, and a failed method is retried once to + # recover flaky failures (mirroring BCH's rerun of failed tests). $isoGroups = @() foreach ($cu in $codeunits) { $cid = "$($cu.Id)" @@ -845,7 +702,7 @@ function Invoke-AlToolTestRun { } } if ($isoGroups.Count -gt 0) { - Write-Host ("isolated fallback/rerun: {0} codeunit(s) (each in its own session)" -f $isoGroups.Count) + Write-Host ("rerun pass: {0} codeunit(s) with unreported or failed method(s) (each in its own session)" -f $isoGroups.Count) foreach ($g in $isoGroups) { $iso = Invoke-AlRunTestsForCodeunit -CodeunitId $g.Id -Methods $g.Methods ` -ProjectPath $projectPath -Company $company -Tenant $tenant -Connection $connection @@ -918,6 +775,5 @@ function Invoke-AlToolTestRun { } Export-ModuleMember -Function Install-AlTool, Get-AlToolConnection, New-AlToolProject, Get-AlToolCompany, ` - Get-DisabledTestKeySet, Get-AlToolTestCodeunits, ConvertFrom-AlRunTestsOutput, ConvertFrom-AlBatchOutput, ` - ConvertTo-AlTestPlanJson, Invoke-AlRunTestsForCodeunit, Invoke-AlBatchRunTests, ` - Add-JUnitTestSuite, Invoke-AlToolTestRun + Get-DisabledTestKeySet, Get-AlToolTestCodeunits, ConvertFrom-AlRunTestsOutput, ` + Invoke-AlRunTestsForCodeunit, Add-JUnitTestSuite, Invoke-AlToolTestRun diff --git a/Actions/RunTests/README.md b/Actions/RunTests/README.md index 9bebfd64ac..d9ed9be4c9 100644 --- a/Actions/RunTests/README.md +++ b/Actions/RunTests/README.md @@ -8,7 +8,7 @@ Only normal tests (testFolders) are handled here. BCPT and page scripting tests ## Test runner -By default this action runs the tests through Microsoft's headless `al runtests` (AlTool) runner. It resolves the kept-alive container's connection settings (server, instance and developer-services port) from the container name, installs the AL developer tools as a `dotnet` global tool (`dotnet tool install Microsoft.Dynamics.BusinessCentral.Development.Tools --prerelease`), enumerates the test codeunits and runs them against the container, then writes the same `TestResults.xml` (JUnit) schema BcContainerHelper produces so downstream test result analysis is unchanged. +By default this action runs the tests through Microsoft's headless `al runtests` (AlTool) runner. It resolves the kept-alive container's connection settings (server, instance and developer-services port) from the container name, installs the AL developer tools as a `dotnet` global tool (`dotnet tool install Microsoft.Dynamics.BusinessCentral.Development.Tools --prerelease`), enumerates the test codeunits and runs each one in its own `al runtests ` invocation, then writes the same `TestResults.xml` (JUnit) schema BcContainerHelper produces so downstream test result analysis is unchanged. To run the tests through BcContainerHelper (or any other runner) instead, add a `RunTestsInBcContainer` override script under the project's `.AL-Go` folder. When present, it fully replaces the built-in AlTool runner and is called once per test app with the same parameters BcContainerHelper's `Run-TestsInBcContainer` expects. diff --git a/RELEASENOTES.md b/RELEASENOTES.md index cc6c42e8a4..3bc9d2ec09 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -2,7 +2,7 @@ A new `useSeparateTestAction` setting (default `false`) lets you move normal test execution (`testFolders`) out of the `RunPipeline` action and into a new dedicated `RunTests` action. When enabled, `RunPipeline` still compiles, publishes and installs the apps and keeps the build container alive, but does not run the normal tests. A new `RunTests` action then runs the tests against that same container and produces the same `TestResults.xml`. Only normal tests are affected; BCPT and page scripting tests are still executed by `RunPipeline`. When the setting is `false`, behavior is unchanged. -By default, the `RunTests` action runs the tests through Microsoft's headless `al runtests` (AlTool) runner instead of BcContainerHelper. It resolves the kept-alive container's connection settings, installs the AL developer tools (`dotnet tool install --prerelease`) and emits the same `TestResults.xml` schema, so downstream test result analysis is unchanged. To run the tests through BcContainerHelper (or any other runner) instead, supply a `RunTestsInBcContainer` override script; it fully replaces the built-in AlTool runner. +By default, the `RunTests` action runs the tests through Microsoft's headless `al runtests` (AlTool) runner instead of BcContainerHelper. It resolves the kept-alive container's connection settings, installs the AL developer tools (`dotnet tool install --prerelease`), runs each test codeunit in its own `al runtests` invocation and emits the same `TestResults.xml` schema, so downstream test result analysis is unchanged. To run the tests through BcContainerHelper (or any other runner) instead, supply a `RunTestsInBcContainer` override script; it fully replaces the built-in AlTool runner. > [!NOTE] > The built-in AlTool runner does not run `Legacy` test-type codeunits or tests that require UI or client-callback interaction. Projects that rely on those should supply a `RunTestsInBcContainer` override script to run their tests through BcContainerHelper instead. diff --git a/Tests/AlToolTestRunner.Test.ps1 b/Tests/AlToolTestRunner.Test.ps1 index 693be26172..5cf98dd63b 100644 --- a/Tests/AlToolTestRunner.Test.ps1 +++ b/Tests/AlToolTestRunner.Test.ps1 @@ -53,53 +53,6 @@ Describe 'AlToolTestRunner.psm1 Tests' { } } - Context 'ConvertFrom-AlBatchOutput' { - It 'Splits batched output into per-codeunit result maps' { - $lines = @( - '===== Codeunit 130001 =====', - 'Results:', - 'PASS TestA (3ms)', - '===== Codeunit 130002 =====', - 'Results:', - 'FAIL TestB (7ms)', - ' boom', - 'AL Callstack:', - ' stack' - ) - - $byCodeunit = ConvertFrom-AlBatchOutput -OutputLines $lines - - $byCodeunit.Keys.Count | Should -Be 2 - $byCodeunit['130001']['TestA'].Outcome | Should -Be 'Pass' - $byCodeunit['130002']['TestB'].Outcome | Should -Be 'Fail' - } - } - - Context 'ConvertTo-AlTestPlanJson' { - It 'Emits arrays at both levels for a single codeunit with a single method' { - $json = ConvertTo-AlTestPlanJson -Groups @(@{ Id = '130001'; Methods = @('OnlyTest') }) - - $json | Should -Match '^\[\{' - $json | Should -Match '\}\]$' - $parsed = $json | ConvertFrom-Json - @($parsed).Count | Should -Be 1 - $parsed[0].codeunitId | Should -Be 130001 - @($parsed[0].testMethods).Count | Should -Be 1 - $parsed[0].testMethods[0] | Should -Be 'OnlyTest' - } - - It 'Serializes multiple codeunits and methods' { - $groups = @( - @{ Id = '1'; Methods = @('A', 'B') }, - @{ Id = '2'; Methods = @('C') } - ) - $parsed = (ConvertTo-AlTestPlanJson -Groups $groups) | ConvertFrom-Json - @($parsed).Count | Should -Be 2 - @($parsed[0].testMethods).Count | Should -Be 2 - $parsed[1].codeunitId | Should -Be 2 - } - } - Context 'Get-DisabledTestKeySet' { It 'Builds per-method keys and whole-codeunit sets from a wildcard' { $disabled = @( From 262babfc669218b263b1c16349bb561de217fa46 Mon Sep 17 00:00:00 2001 From: spetersenms Date: Mon, 10 Aug 2026 12:02:35 +0200 Subject: [PATCH 6/6] Handle array in PS5 --- Actions/RunTests/RunTests.psm1 | 4 ++-- Tests/RunTests.Test.ps1 | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/Actions/RunTests/RunTests.psm1 b/Actions/RunTests/RunTests.psm1 index 925e1994bc..a381dd734a 100644 --- a/Actions/RunTests/RunTests.psm1 +++ b/Actions/RunTests/RunTests.psm1 @@ -44,12 +44,12 @@ function Get-TestAppsToRun { if ($settings.runTestsInAllInstalledTestApps -and $installTestAppsJson -and (Test-Path $installTestAppsJson)) { try { - $installedTestApps = @(Get-Content -Path $installTestAppsJson -Raw | ConvertFrom-Json) + $installedTestApps = Get-Content -Path $installTestAppsJson -Raw | ConvertFrom-Json } catch { throw "Failed to parse JSON file at path '$installTestAppsJson'. Error: $($_.Exception.Message)" } - $testApps += @($installedTestApps | ForEach-Object { $_.TrimStart("(").TrimEnd(")") } | Where-Object { $_ -and (Test-Path $_) }) + $testApps += @($installedTestApps | ForEach-Object { "$_".TrimStart("(").TrimEnd(")") } | Where-Object { $_ -and (Test-Path $_) }) } return @($testApps | Select-Object -Unique) diff --git a/Tests/RunTests.Test.ps1 b/Tests/RunTests.Test.ps1 index 8b72b31c86..318c66a804 100644 --- a/Tests/RunTests.Test.ps1 +++ b/Tests/RunTests.Test.ps1 @@ -72,6 +72,39 @@ Describe 'RunTests.psm1 Tests' { @($testApps).Count | Should -Be 1 Remove-Item -Path $projectPath -Recurse -Force } + + It 'Does not throw and returns compiled test apps when installTestAppsJson is an empty array' { + # Regression: in Windows PowerShell 5.1 ConvertFrom-Json emits a JSON array as a single + # object, so an empty '[]' previously surfaced as a one-element System.Object[] and threw + # "does not contain a method named 'TrimStart'". A test project with no installed test apps + # (the common case) must still return its compiled test apps without throwing. + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @() | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ runTestsInAllInstalledTestApps = $true } + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson + + @($testApps).Count | Should -Be 1 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Includes a single installed test app when runTestsInAllInstalledTestApps is set' { + # Regression: a single-element JSON array must enumerate to the string element (not the + # whole array) on both Windows PowerShell 5.1 and PowerShell 7. + $projectPath = New-TestProject + $installedApp = Join-Path $projectPath 'Installed1.app' + New-Item -Path $installedApp -ItemType File -Force | Out-Null + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @("($installedApp)") | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ runTestsInAllInstalledTestApps = $true } + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson + + @($testApps).Count | Should -Be 1 + $testApps | Should -Contain $installedApp + Remove-Item -Path $projectPath -Recurse -Force + } } Context 'Invoke-AlGoTestRun' {