diff --git a/Actions/.Modules/ReadSettings.psm1 b/Actions/.Modules/ReadSettings.psm1 index 8109baf299..cefea3ee0b 100644 --- a/Actions/.Modules/ReadSettings.psm1 +++ b/Actions/.Modules/ReadSettings.psm1 @@ -174,6 +174,7 @@ function GetDefaultSettings "doNotBuildTests" = $false "doNotPerformUpgrade" = $false "doNotRunTests" = $false + "useSeparateTestAction" = $false "doNotRunBcptTests" = $false "doNotRunPageScriptingTests" = $false "doNotPublishApps" = $false diff --git a/Actions/.Modules/settings.schema.json b/Actions/.Modules/settings.schema.json index 4a1cd415d3..fe8e272767 100644 --- a/Actions/.Modules/settings.schema.json +++ b/Actions/.Modules/settings.schema.json @@ -344,6 +344,10 @@ "doNotRunTests": { "type": "boolean" }, + "useSeparateTestAction": { + "type": "boolean", + "description": "PREVIEW: When set to true, normal tests (testFolders) are no longer executed inside the RunPipeline action. Instead, RunPipeline keeps the build container alive and a separate RunTests action executes the tests afterwards. See https://aka.ms/ALGoSettings#useSeparateTestAction" + }, "doNotRunBcptTests": { "type": "boolean" }, diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index 07c125d2cf..34dcc1bf61 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -19,6 +19,22 @@ Param( [string] $previousAppsPath = '' ) +function New-KeepAliveContainerCredential { + <# + .SYNOPSIS + Generates a credential used to create a build container that is kept alive for the RunTests action. + .DESCRIPTION + When useSeparateTestAction is enabled, RunPipeline keeps the build container alive so the RunTests + action can run tests against it. BcContainerHelper requires an explicit credential when a container + is kept (otherwise it is created with a random password that cannot be reused). This function returns + a PSCredential with a randomly generated complex password. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'A container password must be generated as plain text to build a reusable credential')] + param() + $password = "Pass!$([GUID]::NewGuid().ToString())" + return (New-Object pscredential 'admin', (ConvertTo-SecureString -String $password -AsPlainText -Force)) +} + $containerBaseFolder = $null $projectPath = $null @@ -473,6 +489,36 @@ try { $runAlPipelineParams["preprocessorsymbols"] = $settings.preprocessorSymbols $runAlPipelineParams["features"] = $settings.features + # When useSeparateTestAction is enabled, the normal tests (testFolders) are run by the separate + # RunTests action instead of here, and the build container is kept alive for it. BCPT and page + # scripting tests are unaffected. This needs a build container, which only exists when apps are + # published (doNotPublishApps not set) and the build does not target an online environment; + # otherwise the tests are run here as usual. + $createsTestContainer = (-not $settings.doNotPublishApps) -and -not ($authContext -and $environmentName) + $keepContainerForSeparateTestAction = $false + if ($settings.useSeparateTestAction -and $createsTestContainer) { + Write-Host "useSeparateTestAction is enabled: skipping normal test execution in RunPipeline and keeping the container alive for the RunTests action" + $runAlPipelineParams["doNotRunTests"] = $true + $keepContainerForSeparateTestAction = $true + + # A kept-alive container needs an explicit credential so the RunTests action can reconnect to it. + # Generate one, pass it to Run-AlPipeline, and surface it (masked, base64 JSON) via containerCredential. + if (-not $runAlPipelineParams.ContainsKey('credential')) { + $containerCredential = New-KeepAliveContainerCredential + $runAlPipelineParams["credential"] = $containerCredential + + $containerCredentialPassword = $containerCredential.GetNetworkCredential().Password + $containerCredentialJson = @{ "username" = $containerCredential.UserName; "password" = $containerCredentialPassword } | ConvertTo-Json -Compress + $containerCredentialBase64 = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($containerCredentialJson)) + Write-Host "::add-mask::$containerCredentialPassword" + Write-Host "::add-mask::$containerCredentialBase64" + Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "containerCredential=$containerCredentialBase64" + } + } + elseif ($settings.useSeparateTestAction) { + Write-Host "::Notice::useSeparateTestAction is enabled, but no build container is created for this project (doNotPublishApps is set or the build targets an online environment), so the RunTests action has no container to run tests against and will be skipped." + } + Write-Host "Invoke Run-AlPipeline with buildmode $buildMode" Run-AlPipeline @runAlPipelineParams ` -accept_insiderEula ` @@ -518,6 +564,7 @@ try { -pageScriptingTestResultsFolder (Join-Path $buildArtifactFolder 'PageScriptingTestResultDetails') ` -CreateRuntimePackages:$CreateRuntimePackages ` -appVersion ($versionNumber.MajorMinorVersion) -appBuild ($versionNumber.BuildNumber) -appRevision ($versionNumber.RevisionNumber) ` + -keepContainer:$keepContainerForSeparateTestAction ` -uninstallRemovedApps if ($containerBaseFolder) { diff --git a/Actions/RunTests/AlToolTestRunner.psm1 b/Actions/RunTests/AlToolTestRunner.psm1 new file mode 100644 index 0000000000..b43636dbbd --- /dev/null +++ b/Actions/RunTests/AlToolTestRunner.psm1 @@ -0,0 +1,779 @@ +<# +.SYNOPSIS + Built-in test runner that drives Microsoft's headless `al runtests` (altool) CLI instead of + BcContainerHelper's client-session runner, while producing the same JUnit XML the AL-Go + pipeline already consumes downstream. + +.DESCRIPTION + This module is the default test runner used by the RunTests action (Invoke-AlGoTestRun) when no + RunTestsInBcContainer override script is supplied. For a single test app (identified by + extensionId) it: + 1. Ensures the `al` CLI is installed (dotnet global tool, prerelease). + 2. Resolves the kept-alive container's on-prem connection settings (server/instance/port) + host-side from the container name via BcContainerHelper, and generates a throw-away AL + project with a launch.json so `al runtests` has connection settings. + 3. Enumerates the app's test codeunits + methods via Get-TestsFromBcContainer. + 4. Runs `al runtests --testmethods` once per test codeunit (each in its own + session), with a rerun pass that retries any method that produced no result or a failure. + 5. Emits a JUnit results file matching the exact schema BcContainerHelper produces, so the + downstream AnalyzeTests step keeps working unchanged. + + Credentials are taken from the parameters' PSCredential and exposed to `al` through the + BC_SERVER_USERNAME / BC_SERVER_PASSWORD environment variables (the only auth mechanism the CLI + supports for on-prem UserPassword). + + Known altool output quirks handled here: + - The Results: block emits a phantom `PASS OnRun (..)` trigger entry and a trailing + empty-named aggregate entry per codeunit; both are dropped so counts match the real methods. + - Failure text is the indented lines after `FAIL (Nms)` up to `AL Callstack:`; the + callstack follows until the next result line. +#> + +$ErrorActionPreference = "Stop" + +$script:AlToolPackageId = "Microsoft.Dynamics.BusinessCentral.Development.Tools" + +<# +.SYNOPSIS + Ensures the `al` CLI is available on PATH, installing the prerelease dotnet global tool. +.DESCRIPTION + Installs (or, when already present, leaves in place) the AL developer tools as a dotnet global + tool. The install is guarded by a named mutex so concurrent jobs on the same runner do not + collide on the shared tools store, and availability is re-checked after acquiring the mutex. +.OUTPUTS + [string] The resolved `al` version string. +#> +function Install-AlTool { + param( + [switch] $Force + ) + + $toolsPath = Join-Path $env:USERPROFILE ".dotnet\tools" + if ($env:HOME -and -not $env:USERPROFILE) { + $toolsPath = Join-Path $env:HOME ".dotnet/tools" + } + if (($env:PATH -split [System.IO.Path]::PathSeparator) -notcontains $toolsPath) { + $env:PATH = "$env:PATH$([System.IO.Path]::PathSeparator)$toolsPath" + } + + # Serialize install/update across processes with a named mutex and re-check availability after + # acquiring it (another job may have just installed it). + $mutex = New-Object System.Threading.Mutex($false, "Global\AL-Go-AlTool-Install") + $acquired = $false + try { + try { $acquired = $mutex.WaitOne([TimeSpan]::FromMinutes(10)) } catch [System.Threading.AbandonedMutexException] { $acquired = $true } + + $alAvailable = $null -ne (Get-Command al -ErrorAction SilentlyContinue) + + if (-not $alAvailable) { + Write-Host "Installing '$script:AlToolPackageId' (prerelease) as a dotnet global tool..." + & dotnet tool install $script:AlToolPackageId --global --prerelease *>&1 | ForEach-Object { Write-Host $_ } + if ($LASTEXITCODE -ne 0) { + # A concurrent job may have installed it first; treat as success if `al` now resolves, + # otherwise fall back to an update. + if ($null -eq (Get-Command al -ErrorAction SilentlyContinue)) { + & dotnet tool update $script:AlToolPackageId --global --prerelease *>&1 | ForEach-Object { Write-Host $_ } + } + } + } + elseif ($Force) { + # Explicit opt-in moves to the newest prerelease once, under the mutex. + try { + & dotnet tool update $script:AlToolPackageId --global --prerelease *>&1 | ForEach-Object { Write-Host $_ } + } + catch { + Write-Host "WARNING: 'al' update check failed ($($_.Exception.Message)). Using existing version." + } + } + } + finally { + if ($acquired) { $mutex.ReleaseMutex() } + $mutex.Dispose() + } + + if (-not (Get-Command al -ErrorAction SilentlyContinue)) { + throw "The 'al' CLI is not available after installation. Ensure '$toolsPath' is on PATH and that the runner can reach nuget.org." + } + + $version = (& al --version 2>&1 | Select-Object -First 1) + Write-Host "Using al CLI version: $version" + return "$version" +} + +<# +.SYNOPSIS + Resolves the on-prem connection settings (server URL, instance, dev-service port) for a container. +.DESCRIPTION + BcContainerHelper's Run-TestsInBcContainer only needs the container name and resolves the + endpoint internally, but `al runtests` needs an explicit server/instance/port. This reads them + host-side from the container's server configuration, falling back to conventional defaults. +.PARAMETER ContainerName + The name of the build container. +.OUTPUTS + [hashtable] @{ Server; ServerInstance; Port } +#> +function Get-AlToolConnection { + param( + [Parameter(Mandatory = $true)][string] $ContainerName + ) + + $server = "http://$ContainerName" + $instance = "BC" + $port = 7049 + + try { + $config = Get-BcContainerServerConfiguration -ContainerName $ContainerName + if ($config) { + if ($config.ServerInstance) { $instance = "$($config.ServerInstance)" } + if ($config.DeveloperServicesPort) { $port = [int]$config.DeveloperServicesPort } + } + } + catch { + Write-Host "WARNING: Could not read server configuration for '$ContainerName' ($($_.Exception.Message)). Falling back to $server/${instance}:$port." + } + + return @{ Server = $server; ServerInstance = $instance; Port = $port } +} + +<# +.SYNOPSIS + Creates a throw-away AL project folder with a launch.json targeting the container so `al runtests` + can resolve connection settings. +.PARAMETER ContainerName + The name of the build container. +.PARAMETER Tenant + The tenant to connect to. +.PARAMETER Connection + The connection hashtable produced by Get-AlToolConnection. +.OUTPUTS + [string] Path to the generated project folder. +#> +function New-AlToolProject { + param( + [Parameter(Mandatory = $true)][string] $ContainerName, + [Parameter(Mandatory = $true)][string] $Tenant, + [Parameter(Mandatory = $true)][hashtable] $Connection + ) + + $projectRoot = Join-Path ([System.IO.Path]::GetTempPath()) "altool-project-$ContainerName" + $vscodeDir = Join-Path $projectRoot ".vscode" + New-Item -ItemType Directory -Path $vscodeDir -Force | Out-Null + + $appJson = [ordered]@{ + id = [System.Guid]::NewGuid().ToString() + name = "AlToolTestDriver" + publisher = "AL-Go" + version = "1.0.0.0" + platform = "1.0.0.0" + runtime = "15.0" + } + $appJson | ConvertTo-Json | Set-Content -Path (Join-Path $projectRoot "app.json") -Encoding UTF8 + + $launch = [ordered]@{ + version = "0.2.0" + configurations = @( + [ordered]@{ + name = "altool" + type = "al" + request = "launch" + server = $Connection.Server + serverInstance = $Connection.ServerInstance + port = $Connection.Port + tenant = $Tenant + authentication = "UserPassword" + environmentType = "OnPrem" + startupObjectId = 22 + startupObjectType = "Page" + schemaUpdateMode = "Synchronize" + } + ) + } + $launch | ConvertTo-Json -Depth 5 | Set-Content -Path (Join-Path $vscodeDir "launch.json") -Encoding UTF8 + + return $projectRoot +} + +<# +.SYNOPSIS + Resolves the company `al runtests` should target. +.DESCRIPTION + Honors an explicitly requested company name first, then falls back to the container's default + (preferring an evaluation company) via Get-CompanyInBcContainer. +.PARAMETER ContainerName + The name of the build container. +.PARAMETER Tenant + The tenant to connect to. +.PARAMETER CompanyName + The company name requested by the caller (optional). +.OUTPUTS + [string] Company name, or empty string if none could be resolved. +#> +function Get-AlToolCompany { + param( + [Parameter(Mandatory = $true)][string] $ContainerName, + [Parameter(Mandatory = $true)][string] $Tenant, + [string] $CompanyName = "" + ) + + if (-not [string]::IsNullOrWhiteSpace($CompanyName)) { + return $CompanyName + } + + try { + $companies = @(Get-CompanyInBcContainer -containerName $ContainerName -tenant $Tenant) + if ($companies.Count -gt 0) { + $preferred = $companies | Where-Object { $_.evaluationCompany -eq $true } | Select-Object -First 1 + $company = if ($preferred) { $preferred.companyName } else { $companies[0].companyName } + return "$company" + } + } + catch { + Write-Host "WARNING: Could not enumerate companies for '$ContainerName' ($($_.Exception.Message))." + } + return "" +} + +<# +.SYNOPSIS + Builds disabled-test lookups from the disabledTests list: per-method keys plus whole-codeunit + names (where a `*` wildcard method disables the entire codeunit). +.DESCRIPTION + Each disabledTests entry has a codeunitName and either a single 'method', an array of methods, or + the wildcard '*'. A '*' entry means the ENTIRE codeunit is disabled, so it must exclude every + method of that codeunit - not a literal method named '*'. Names/keys are lowercased for + case-insensitive matching. +.PARAMETER DisabledTests + Array of disabled-test entries. +.OUTPUTS + [hashtable] @{ Methods = ::">; Codeunits = "> } +#> +function Get-DisabledTestKeySet { + param( + [array] $DisabledTests = @() + ) + + $methodSet = @{} + $codeunitSet = @{} + foreach ($entry in $DisabledTests) { + if (-not $entry) { continue } + $cuName = "$($entry.codeunitName)".ToLowerInvariant() + $methods = @() + if ($entry.PSObject.Properties['method'] -and $entry.method) { $methods = @($entry.method) } + foreach ($m in $methods) { + if ("$m" -eq '*') { + $codeunitSet[$cuName] = $true + } + else { + $methodSet["$cuName::$("$m".ToLowerInvariant())"] = $true + } + } + } + return @{ Methods = $methodSet; Codeunits = $codeunitSet } +} + +<# +.SYNOPSIS + Enumerates the test codeunits + enabled methods for an app in the container. +.DESCRIPTION + Uses Get-TestsFromBcContainer with the app's extensionId to list every test codeunit and method. + When a disabledTests list is supplied, disabled methods (and whole codeunits marked with a `*` + wildcard) are filtered out here, because `al runtests` has no equivalent of BCH's run-time + DisableTestMethod control. +.PARAMETER Parameters + Hashtable with containerName, tenant, credential, extensionId and optionally testType and + disabledTests. +.OUTPUTS + [object[]] Codeunit objects with .Id, .Name, .Tests (enabled method name array). +#> +function Get-AlToolTestCodeunits { + param( + [Parameter(Mandatory = $true)][hashtable] $Parameters + ) + + $getTestsParams = @{ + containerName = $Parameters.containerName + tenant = if ($Parameters.ContainsKey("tenant") -and $Parameters.tenant) { $Parameters.tenant } else { "default" } + credential = $Parameters.credential + extensionId = $Parameters.extensionId + ignoreGroups = $true + } + if ($Parameters.ContainsKey("testType") -and $Parameters.testType) { + $getTestsParams.testType = $Parameters.testType + } + + $codeunits = @(Get-TestsFromBcContainer @getTestsParams) + + $disabledMethods = @{} + $disabledCodeunits = @{} + if ($Parameters.ContainsKey("disabledTests") -and $Parameters.disabledTests) { + $lookup = Get-DisabledTestKeySet -DisabledTests @($Parameters.disabledTests) + $disabledMethods = $lookup.Methods + $disabledCodeunits = $lookup.Codeunits + } + + $result = @() + $disabledCount = 0 + foreach ($cu in $codeunits) { + $cuNameLower = "$($cu.Name)".ToLowerInvariant() + $methods = @($cu.Tests | ForEach-Object { "$_" }) + + if ($disabledCodeunits.ContainsKey($cuNameLower)) { + $disabledCount += $methods.Count + continue + } + + if ($disabledMethods.Count -gt 0) { + $enabled = @($methods | Where-Object { -not $disabledMethods.ContainsKey("$cuNameLower::$("$_".ToLowerInvariant())") }) + $disabledCount += ($methods.Count - $enabled.Count) + $methods = $enabled + } + if ($methods.Count -gt 0) { + $result += [PSCustomObject]@{ Id = $cu.Id; Name = $cu.Name; Tests = $methods } + } + } + + if ($disabledCount -gt 0) { + Write-Host "Excluded $disabledCount disabled test method(s) from altool enumeration." + } + return @($result) +} + +<# +.SYNOPSIS + Parses the `Results:` block of a single `al runtests` invocation into per-method outcomes. +.DESCRIPTION + Drops the phantom `OnRun` trigger entry and the trailing empty-named aggregate entry. Captures + the failure message (lines up to `AL Callstack:`) and callstack (following lines) for failures. +.PARAMETER OutputLines + The lines of `al runtests --raw` output. +.OUTPUTS + [hashtable] method name -> @{ Outcome (Pass/Fail/Skip); Ms; Message; Stacktrace } +#> +function ConvertFrom-AlRunTestsOutput { + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][AllowEmptyString()][string[]] $OutputLines + ) + + $results = @{} + $resultLineRegex = '^\s*(PASS|FAIL|SKIP)\s+(.*?)\s*\((\d+)ms\)\s*$' + + $startIdx = -1 + for ($i = 0; $i -lt $OutputLines.Count; $i++) { + if ($OutputLines[$i] -match '^\s*Results:\s*$') { $startIdx = $i + 1; break } + } + if ($startIdx -lt 0) { return $results } + + $i = $startIdx + while ($i -lt $OutputLines.Count) { + $line = $OutputLines[$i] + $m = [regex]::Match($line, $resultLineRegex) + if (-not $m.Success) { $i++; continue } + + $outcome = switch ($m.Groups[1].Value) { 'PASS' { 'Pass' } 'FAIL' { 'Fail' } 'SKIP' { 'Skip' } } + $name = $m.Groups[2].Value.Trim() + $ms = [int]$m.Groups[3].Value + + if ([string]::IsNullOrWhiteSpace($name) -or $name -eq 'OnRun') { $i++; continue } + + $message = '' + $stackText = '' + if ($outcome -eq 'Fail') { + $msgLines = @() + $stackLines = @() + $inStack = $false + $j = $i + 1 + while ($j -lt $OutputLines.Count) { + $next = $OutputLines[$j] + if ([regex]::IsMatch($next, $resultLineRegex)) { break } + if ($next -match '^\s*AL Callstack:\s*$') { $inStack = $true; $j++; continue } + if ($inStack) { + if ($next.Trim().Length -gt 0) { $stackLines += $next.Trim() } + } + else { + if ($next.Trim().Length -gt 0) { $msgLines += $next.Trim() } + } + $j++ + } + $message = ($msgLines -join ' ').Trim() + $stackText = ($stackLines -join ';') + $i = $j + } + else { + $i++ + } + + $results[$name] = @{ Outcome = $outcome; Ms = $ms; Message = $message; Stacktrace = $stackText } + } + + return $results +} + +<# +.SYNOPSIS + Runs `al runtests` for one codeunit and returns the parsed per-method results plus raw output. +.PARAMETER CodeunitId + The codeunit id to run. +.PARAMETER Methods + The test method names to run. +.PARAMETER ProjectPath + The throw-away AL project folder. +.PARAMETER Company + The company to run against. +.PARAMETER Tenant + The tenant to connect to. +.PARAMETER Connection + The connection hashtable produced by Get-AlToolConnection. +.OUTPUTS + [hashtable] @{ Results (method->outcome map); ElapsedSec; Raw; Connected (bool) } +#> +function Invoke-AlRunTestsForCodeunit { + param( + [Parameter(Mandatory = $true)][string] $CodeunitId, + [Parameter(Mandatory = $true)][string[]] $Methods, + [Parameter(Mandatory = $true)][string] $ProjectPath, + [Parameter(Mandatory = $true)][string] $Company, + [Parameter(Mandatory = $true)][string] $Tenant, + [Parameter(Mandatory = $true)][hashtable] $Connection + ) + + # `al runtests` emits structured JSON by default in newer builds; `--raw` restores the + # human-readable text summary ("Test run completed: ..." + a "Results:" block of + # "PASS|FAIL|SKIP (Nms)" lines) that ConvertFrom-AlRunTestsOutput parses. + $alArgs = @( + 'runtests', $CodeunitId, + '--project', $ProjectPath, + '--company', $Company, + '--server', $Connection.Server, + '--serverinstance', $Connection.ServerInstance, + '--port', "$($Connection.Port)", + '--environmenttype', 'OnPrem', + '--authentication', 'UserPassword', + '--tenant', $Tenant, + '--raw', + '--testmethods' + ) + $Methods + + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $output = & al @alArgs 2>&1 + $sw.Stop() + + $lines = @($output | ForEach-Object { "$_" }) + $connected = ($lines | Where-Object { $_ -match 'Test run completed:' }).Count -gt 0 + $parsed = ConvertFrom-AlRunTestsOutput -OutputLines $lines + + if ($connected -and $parsed.Count -eq 0 -and $Methods.Count -gt 0) { + Write-Host "::warning::al runtests connected for codeunit $CodeunitId but produced no parseable results. Raw output follows (possible output-format change):" + Write-Host ($lines -join "`n") + } + + return @{ + Results = $parsed + ElapsedSec = [Math]::Round($sw.Elapsed.TotalSeconds, 3) + Raw = ($lines -join "`n") + Connected = $connected + } +} + +<# +.SYNOPSIS + Appends a JUnit for one codeunit to the given document, matching the + exact schema BcContainerHelper produces. +.PARAMETER Doc + The JUnit XmlDocument being built. +.PARAMETER TestSuitesNode + The root element to append to. +.PARAMETER Codeunit + The codeunit object (.Id, .Name). +.PARAMETER RequestedMethods + The method names that were requested for this codeunit. +.PARAMETER MethodResults + The parsed per-method result map for this codeunit. +.PARAMETER ExtensionId + The extension (app) id. +.PARAMETER AppName + The app name. +.PARAMETER Hostname + The runner host name. +.PARAMETER ElapsedSec + The elapsed time (seconds) attributed to this codeunit. +.OUTPUTS + [int] Number of failing methods in this codeunit. +#> +function Add-JUnitTestSuite { + param( + [Parameter(Mandatory = $true)][System.Xml.XmlDocument] $Doc, + [Parameter(Mandatory = $true)][System.Xml.XmlElement] $TestSuitesNode, + [Parameter(Mandatory = $true)] $Codeunit, + [Parameter(Mandatory = $true)][string[]] $RequestedMethods, + [Parameter(Mandatory = $true)][hashtable] $MethodResults, + [Parameter(Mandatory = $true)][string] $ExtensionId, + [Parameter(Mandatory = $true)][string] $AppName, + [Parameter(Mandatory = $true)][string] $Hostname, + [Parameter(Mandatory = $true)][double] $ElapsedSec + ) + + $ci = [System.Globalization.CultureInfo]::InvariantCulture + $suiteName = "$($Codeunit.Id) $($Codeunit.Name)" + + $suite = $Doc.CreateElement("testsuite") + $suite.SetAttribute("name", $suiteName) + $suite.SetAttribute("timestamp", (Get-Date -Format s)) + $suite.SetAttribute("hostname", $Hostname) + + $props = $Doc.CreateElement("properties") + $suite.AppendChild($props) | Out-Null + $extProp = $Doc.CreateElement("property") + $extProp.SetAttribute("name", "extensionid") + $extProp.SetAttribute("value", $ExtensionId) + $props.AppendChild($extProp) | Out-Null + if ($AppName) { + $appProp = $Doc.CreateElement("property") + $appProp.SetAttribute("name", "appName") + $appProp.SetAttribute("value", $AppName) + $props.AppendChild($appProp) | Out-Null + } + + $failed = 0 + $skipped = 0 + foreach ($method in $RequestedMethods) { + $res = $MethodResults[$method] + + $tc = $Doc.CreateElement("testcase") + $tc.SetAttribute("classname", $suiteName) + $tc.SetAttribute("name", $method) + + if ($null -eq $res) { + # Method was requested but the runner produced no result (e.g. connect failure) -> error. + $tc.SetAttribute("time", "0") + $failure = $Doc.CreateElement("failure") + $failure.SetAttribute("message", "No result produced by al runtests") + $failure.InnerText = "" + $tc.AppendChild($failure) | Out-Null + $failed++ + } + else { + $tc.SetAttribute("time", ([Math]::Round($res.Ms / 1000.0, 3)).ToString($ci)) + switch ($res.Outcome) { + 'Fail' { + $failure = $Doc.CreateElement("failure") + $failure.SetAttribute("message", "$($res.Message)") + $failure.InnerText = "$($res.Stacktrace)".Replace(";", "`n") + $tc.AppendChild($failure) | Out-Null + $failed++ + } + 'Skip' { + $sk = $Doc.CreateElement("skipped") + $tc.AppendChild($sk) | Out-Null + $skipped++ + } + } + } + $suite.AppendChild($tc) | Out-Null + } + + $suite.SetAttribute("tests", "$($RequestedMethods.Count)") + $suite.SetAttribute("errors", "0") + $suite.SetAttribute("failures", "$failed") + $suite.SetAttribute("skipped", "$skipped") + $suite.SetAttribute("time", ([Math]::Round($ElapsedSec, 3)).ToString($ci)) + + $TestSuitesNode.AppendChild($suite) | Out-Null + return $failed +} + +<# +.SYNOPSIS + Runs all of a single app's test codeunits through `al runtests` and writes a JUnit results file. +.DESCRIPTION + The built-in default test runner for the RunTests action. Expects $Parameters to contain + containerName, credential, extensionId and (optionally) tenant, companyName, appName, + disabledTests and JUnitResultFileName. Runs each of the app's test codeunits in its own + `al runtests` invocation, then re-runs any method that produced no result or a failure once, + and appends a BcContainerHelper-schema JUnit result. Returns whether every executed method + passed. +.PARAMETER Parameters + The BcContainerHelper-shaped test parameters built by Invoke-AlGoTestRun. +.OUTPUTS + [bool] $true if all executed methods passed; $false otherwise. +#> +function Invoke-AlToolTestRun { + param( + [Parameter(Mandatory = $true)][hashtable] $Parameters + ) + + Install-AlTool | Out-Null + + $containerName = $Parameters.containerName + $tenant = if ($Parameters.ContainsKey("tenant") -and $Parameters.tenant) { "$($Parameters.tenant)" } else { "default" } + $extensionId = "$($Parameters.extensionId)" + $appName = if ($Parameters.ContainsKey("appName")) { "$($Parameters.appName)" } else { "" } + $companyName = if ($Parameters.ContainsKey("companyName")) { "$($Parameters.companyName)" } else { "" } + + if ([string]::IsNullOrWhiteSpace($extensionId)) { + throw "Invoke-AlToolTestRun requires 'extensionId' in parameters." + } + + # Expose credentials to the al CLI (only auth channel it supports for on-prem UserPassword). + # Set inside the try below so the finally always clears them from the process environment. + if ($Parameters.credential -isnot [System.Management.Automation.PSCredential]) { + throw "Invoke-AlToolTestRun requires a PSCredential in parameters.credential." + } + + try { + $env:BC_SERVER_USERNAME = $Parameters.credential.UserName + $env:BC_SERVER_PASSWORD = $Parameters.credential.GetNetworkCredential().Password + + $connection = Get-AlToolConnection -ContainerName $containerName + $projectPath = New-AlToolProject -ContainerName $containerName -Tenant $tenant -Connection $connection + $company = Get-AlToolCompany -ContainerName $containerName -Tenant $tenant -CompanyName $companyName + if ([string]::IsNullOrWhiteSpace($company)) { + throw "Could not resolve a company to run tests against in container '$containerName'." + } + + Write-Host "altool run: app='$appName' extensionId=$extensionId company='$company' server='$($connection.Server)' instance='$($connection.ServerInstance)' port=$($connection.Port) tenant='$tenant'" + + $codeunits = @(Get-AlToolTestCodeunits -Parameters $Parameters) + Write-Host "Enumerated $($codeunits.Count) test codeunit(s) for app '$appName'." + if ($codeunits.Count -eq 0) { + Write-Host "No test codeunits to run for app '$appName'; nothing to do." + return $true + } + + $hostname = [System.Net.Dns]::GetHostName() + + # Append to an existing JUnit file when present (Invoke-AlGoTestRun runs one app at a time + # into the same TestResults.xml), matching BCH's AppendToJUnitResultFile behavior. + $junitFile = if ($Parameters.ContainsKey("JUnitResultFileName")) { $Parameters.JUnitResultFileName } else { "" } + $doc = New-Object System.Xml.XmlDocument + $suites = $null + if (-not [string]::IsNullOrWhiteSpace($junitFile) -and (Test-Path $junitFile)) { + try { + $doc.Load($junitFile) + $suites = $doc.DocumentElement + if (-not $suites -or $suites.LocalName -ne 'testsuites') { $suites = $null; $doc = New-Object System.Xml.XmlDocument } + } + catch { + Write-Host "WARNING: Could not load existing JUnit file '$junitFile' ($($_.Exception.Message)); starting fresh." + $doc = New-Object System.Xml.XmlDocument + $suites = $null + } + } + if (-not $suites) { + $doc.AppendChild($doc.CreateXmlDeclaration("1.0", "UTF-8", $null)) | Out-Null + $suites = $doc.CreateElement("testsuites") + $doc.AppendChild($suites) | Out-Null + } + + $allPassed = $true + $merged = @{} # merged[codeunitId] = @{ method -> result } + $totalElapsed = 0.0 + + # PRIMARY execution: run each test codeunit in its OWN `al runtests --testmethods` + # invocation (a fresh session per codeunit). The official altool has no batch/plan mode, + # so per-codeunit is the only supported execution model. + foreach ($cu in $codeunits) { + $cid = "$($cu.Id)" + $methods = @($cu.Tests | ForEach-Object { "$_" }) + $run = Invoke-AlRunTestsForCodeunit -CodeunitId $cid -Methods $methods ` + -ProjectPath $projectPath -Company $company -Tenant $tenant -Connection $connection + $totalElapsed += [double]$run.ElapsedSec + if (-not $run.Connected) { + Write-Host "::warning::al runtests did not complete for codeunit $cid ('$($cu.Name)') in app '$appName'. Raw output:" + Write-Host $run.Raw + } + $merged[$cid] = $run.Results + } + + # Rerun-on-failure pass. Each affected codeunit runs again in its OWN `al runtests` call + # (a fresh session): a method that produced no result (a state-sensitive codeunit can leave + # a method unreported) is retried for correctness, and a failed method is retried once to + # recover flaky failures (mirroring BCH's rerun of failed tests). + $isoGroups = @() + foreach ($cu in $codeunits) { + $cid = "$($cu.Id)" + $requested = @($cu.Tests | ForEach-Object { "$_" }) + $cuResults = $merged[$cid] + $retryMethods = @($requested | Where-Object { + $r = if ($cuResults) { $cuResults[$_] } else { $null } + ($null -eq $r) -or ($r.Outcome -eq 'Fail') + }) + if ($retryMethods.Count -gt 0) { + $isoGroups += @{ Id = $cid; Methods = $retryMethods; Name = $cu.Name } + } + } + if ($isoGroups.Count -gt 0) { + Write-Host ("rerun pass: {0} codeunit(s) with unreported or failed method(s) (each in its own session)" -f $isoGroups.Count) + foreach ($g in $isoGroups) { + $iso = Invoke-AlRunTestsForCodeunit -CodeunitId $g.Id -Methods $g.Methods ` + -ProjectPath $projectPath -Company $company -Tenant $tenant -Connection $connection + $totalElapsed += [double]$iso.ElapsedSec + if (-not $merged.ContainsKey($g.Id)) { $merged[$g.Id] = @{} } + foreach ($mName in $iso.Results.Keys) { $merged[$g.Id][$mName] = $iso.Results[$mName] } + } + } + + # Distribute the app's REAL al wall-clock across its codeunits weighted by each codeunit's + # method-ms share (al's per-method `ms` under-reports real work, so it is used only for + # weighting, not as the absolute suite time). Equal split as a fallback. + $cuMsShare = @{} + $grandMs = 0.0 + foreach ($cu in $codeunits) { + $cuResults = $merged["$($cu.Id)"] + $ms = 0.0 + if ($cuResults) { foreach ($mName in $cuResults.Keys) { $ms += [double]$cuResults[$mName].Ms } } + $cuMsShare["$($cu.Id)"] = $ms + $grandMs += $ms + } + + # Build one JUnit per codeunit from the merged results. + $idx = 0 + foreach ($cu in $codeunits) { + $idx++ + $methods = @($cu.Tests | ForEach-Object { "$_" }) + $cuResults = $merged["$($cu.Id)"] + if ($null -eq $cuResults) { $cuResults = @{} } + + if ($grandMs -gt 0) { + $suiteSec = $totalElapsed * ($cuMsShare["$($cu.Id)"] / $grandMs) + } + elseif ($codeunits.Count -gt 0) { + $suiteSec = $totalElapsed / $codeunits.Count + } + else { + $suiteSec = 0.0 + } + + $failed = Add-JUnitTestSuite -Doc $doc -TestSuitesNode $suites -Codeunit $cu ` + -RequestedMethods $methods -MethodResults $cuResults -ExtensionId $extensionId ` + -AppName $appName -Hostname $hostname -ElapsedSec $suiteSec + + if ($failed -gt 0) { $allPassed = $false } + + Write-Host ("[{0}/{1}] cu {2} '{3}' -> {4} failed of {5} method(s)" -f ` + $idx, $codeunits.Count, $cu.Id, $cu.Name, $failed, $methods.Count) + } + Write-Host ("Run for app '{0}': {1} codeunit(s) in {2}s real al wall-clock." -f ` + $appName, $codeunits.Count, [Math]::Round($totalElapsed, 2)) + + if (-not [string]::IsNullOrWhiteSpace($junitFile)) { + $dir = [System.IO.Path]::GetDirectoryName($junitFile) + if ($dir -and -not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null } + $doc.Save($junitFile) + Write-Host "Wrote JUnit results for app '$appName' to $junitFile" + } + else { + Write-Host "WARNING: No JUnitResultFileName in parameters; results not persisted for app '$appName'." + } + + return $allPassed + } + finally { + # Do not let the container credential linger in the process environment after the run. + Remove-Item Env:\BC_SERVER_USERNAME -ErrorAction SilentlyContinue + Remove-Item Env:\BC_SERVER_PASSWORD -ErrorAction SilentlyContinue + } +} + +Export-ModuleMember -Function Install-AlTool, Get-AlToolConnection, New-AlToolProject, Get-AlToolCompany, ` + Get-DisabledTestKeySet, Get-AlToolTestCodeunits, ConvertFrom-AlRunTestsOutput, ` + Invoke-AlRunTestsForCodeunit, Add-JUnitTestSuite, Invoke-AlToolTestRun diff --git a/Actions/RunTests/README.md b/Actions/RunTests/README.md new file mode 100644 index 0000000000..d9ed9be4c9 --- /dev/null +++ b/Actions/RunTests/README.md @@ -0,0 +1,38 @@ +# Run tests + +Run the normal tests (testFolders) for an AL-Go project against the build container created and kept alive by the RunPipeline action. + +This action only does anything when the `useSeparateTestAction` setting is enabled. In that case, the RunPipeline action compiles, publishes and installs the apps, skips the normal tests and keeps the build container alive. This action then runs the normal tests against that same container and writes the results to `TestResults.xml` in the project folder. + +Only normal tests (testFolders) are handled here. BCPT and page scripting tests continue to be executed by the RunPipeline action. + +## Test runner + +By default this action runs the tests through Microsoft's headless `al runtests` (AlTool) runner. It resolves the kept-alive container's connection settings (server, instance and developer-services port) from the container name, installs the AL developer tools as a `dotnet` global tool (`dotnet tool install Microsoft.Dynamics.BusinessCentral.Development.Tools --prerelease`), enumerates the test codeunits and runs each one in its own `al runtests ` invocation, then writes the same `TestResults.xml` (JUnit) schema BcContainerHelper produces so downstream test result analysis is unchanged. + +To run the tests through BcContainerHelper (or any other runner) instead, add a `RunTestsInBcContainer` override script under the project's `.AL-Go` folder. When present, it fully replaces the built-in AlTool runner and is called once per test app with the same parameters BcContainerHelper's `Run-TestsInBcContainer` expects. + +### Known limitations + +The built-in AlTool runner does not run `Legacy` test-type codeunits or tests that require UI or client-callback interaction. Repositories that rely on those should run their tests through BcContainerHelper by supplying a `RunTestsInBcContainer` override script (as described above), which fully replaces the AlTool runner. + +## INPUT + +### ENV variables + +| Name | Description | +| :-- | :-- | +| Settings | env.Settings must be set by a prior call to the ReadSettings Action | +| containerName | env.containerName is set by the RunPipeline action and identifies the container to run tests against (the container name is otherwise derived from the project) | + +### Parameters + +| Name | Required | Description | Default value | +| :-- | :-: | :-- | :-- | +| shell | | The shell (powershell or pwsh) in which the PowerShell script should run | powershell | +| project | | Project folder | '.' | +| installTestAppsJson | | Path to a JSON file containing a list of test apps to run tests in | '' | + +## OUTPUT + +None diff --git a/Actions/RunTests/RunTests.ps1 b/Actions/RunTests/RunTests.ps1 new file mode 100644 index 0000000000..e5acf2c7d9 --- /dev/null +++ b/Actions/RunTests/RunTests.ps1 @@ -0,0 +1,92 @@ +Param( + [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] + [string] $token, + [Parameter(HelpMessage = "Project folder", Mandatory = $false)] + [string] $project = "", + [Parameter(HelpMessage = "A path to a JSON-formatted list of test apps to run tests in", Mandatory = $false)] + [string] $installTestAppsJson = '' +) + +<# +.SYNOPSIS + Runs the normal tests (testFolders) for an AL-Go project against the build container + created and kept alive by the RunPipeline action. +.DESCRIPTION + Runs the normal tests (testFolders) of an AL-Go project against the build container that the + RunPipeline action created and kept alive. This runs as part of the build when the + useSeparateTestAction setting is enabled; when it is not, RunPipeline runs the tests instead. + Results are written to TestResults.xml in the project folder. + + Only normal tests (testFolders) are run here. BCPT and page scripting tests are run by the + RunPipeline action. +.PARAMETER token + The GitHub token running the action. It is exposed as the _token environment variable so + downstream test override scripts (for example, BCApps test tolerance, which downloads the + unstable-tests artifact) can authenticate against GitHub. +.PARAMETER project + Project folder. +.PARAMETER installTestAppsJson + A path to a JSON-formatted list of test apps (produced by previous jobs) to run tests in. +.EXAMPLE + RunTests.ps1 -project 'MyProject' +#> + +. (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve) +Import-Module (Join-Path $PSScriptRoot '..\TelemetryHelper.psm1' -Resolve) +Import-Module (Join-Path $PSScriptRoot 'RunTests.psm1' -Resolve) -DisableNameChecking -Force +DownloadAndImportBcContainerHelper + +function Get-TestRunnerCredential { + <# + .SYNOPSIS + Returns the credential used by the test runner to connect to the build container. + .DESCRIPTION + RunPipeline creates the container and keeps it alive when useSeparateTestAction is set. + When RunPipeline surfaces the container credential (masked, as base64-encoded JSON in + the containerCredential environment variable), it is used here so the test runner can + connect to the same container. Otherwise a default credential is used. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'The container credential is surfaced by RunPipeline as plain text')] + param() + if ($ENV:containerCredential) { + $credentialJson = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ENV:containerCredential)) | ConvertFrom-Json + $securePassword = ConvertTo-SecureString -String $credentialJson.password -AsPlainText -Force + return New-Object System.Management.Automation.PSCredential($credentialJson.username, $securePassword) + } + $securePassword = ConvertTo-SecureString -String ([GUID]::NewGuid().ToString()) -AsPlainText -Force + return New-Object System.Management.Automation.PSCredential("admin", $securePassword) +} + +if ($project -eq ".") { $project = "" } + +$baseFolder = $ENV:GITHUB_WORKSPACE +$projectPath = Join-Path $baseFolder $project + +Write-Host "Use settings" +$settings = $env:Settings | ConvertFrom-Json | ConvertTo-HashTable + +# Surface the token so RunTestsInBcContainer override scripts (e.g. BCApps test tolerance) can authenticate against GitHub. +$ENV:_token = $token + +# Analyze the repository to determine the test folders (and other test related settings) +$settings = AnalyzeRepo -settings $settings -baseFolder $baseFolder -project $project -doNotCheckArtifactSetting + +# Resolve the container kept alive by RunPipeline (name is deterministic per project, also exported to the environment). +$containerName = $ENV:containerName +if (-not $containerName) { + $containerName = GetContainerName($project) +} + +# Credentials used to connect to the build container. +$credential = Get-TestRunnerCredential + +# A RunTestsInBcContainer override script, if present, replaces the built-in AlTool test runner. +$overrideParams = Get-ScriptOverrides -ALGoFolderName (Join-Path $projectPath ".AL-Go") -OverrideScriptNames @("RunTestsInBcContainer") + +Invoke-AlGoTestRun ` + -settings $settings ` + -projectPath $projectPath ` + -containerName $containerName ` + -credential $credential ` + -installTestAppsJson $installTestAppsJson ` + -runTestsOverride $overrideParams['RunTestsInBcContainer'] diff --git a/Actions/RunTests/RunTests.psm1 b/Actions/RunTests/RunTests.psm1 new file mode 100644 index 0000000000..a381dd734a --- /dev/null +++ b/Actions/RunTests/RunTests.psm1 @@ -0,0 +1,151 @@ +<# +.SYNOPSIS + Helper module for the RunTests action. +.DESCRIPTION + Contains the logic for running the normal tests (testFolders) of an AL-Go project against a + build container that was created and kept alive by the RunPipeline action. Kept in a module + so the logic can be unit tested independently of the action entry script. + + By default the tests are run through the AlTool (`al runtests`) runner in AlToolTestRunner.psm1. + A RunTestsInBcContainer override script, when supplied, replaces that default with the + BcContainerHelper test runner (this is how, for example, BCApps supplies its own runner). +#> + +Import-Module (Join-Path $PSScriptRoot 'AlToolTestRunner.psm1' -Resolve) -DisableNameChecking -Force + +function Get-TestAppsToRun { + <# + .SYNOPSIS + Determines the set of test app files to run tests in. + .DESCRIPTION + Collects the test apps compiled for the project (found in the build artifacts TestApps + folder) and, when runTestsInAllInstalledTestApps is enabled, the test apps installed from + previous jobs (listed in installTestAppsJson). Test apps wrapped in parentheses are + unwrapped (matching Run-AlPipeline semantics where such apps are otherwise not tested). + .PARAMETER settings + The (analyzed) AL-Go settings hashtable. + .PARAMETER projectPath + The full path to the project folder. + .PARAMETER installTestAppsJson + Path to a JSON file with the list of installed test apps. + #> + Param( + [hashtable] $settings, + [string] $projectPath, + [string] $installTestAppsJson = '' + ) + + $testAppOutputFolder = Join-Path $projectPath ".buildartifacts\TestApps" + + $testApps = @() + if (Test-Path $testAppOutputFolder) { + $testApps += @(Get-ChildItem -Path $testAppOutputFolder -Filter "*.app" -ErrorAction SilentlyContinue | ForEach-Object { $_.FullName }) + } + + if ($settings.runTestsInAllInstalledTestApps -and $installTestAppsJson -and (Test-Path $installTestAppsJson)) { + try { + $installedTestApps = Get-Content -Path $installTestAppsJson -Raw | ConvertFrom-Json + } + catch { + throw "Failed to parse JSON file at path '$installTestAppsJson'. Error: $($_.Exception.Message)" + } + $testApps += @($installedTestApps | ForEach-Object { "$_".TrimStart("(").TrimEnd(")") } | Where-Object { $_ -and (Test-Path $_) }) + } + + return @($testApps | Select-Object -Unique) +} + +function Invoke-AlGoTestRun { + <# + .SYNOPSIS + Runs the normal tests for an AL-Go project against a kept-alive build container. + .DESCRIPTION + Runs tests in each test app against the given container and writes the results to + testResultsFile in JUnit format. Honors the treatTestFailuresAsWarnings setting. By default + the tests are run through the AlTool (`al runtests`) runner. When a RunTestsInBcContainer + override script is provided, it is used instead of the built-in AlTool runner. + .PARAMETER settings + The (analyzed) AL-Go settings hashtable. + .PARAMETER projectPath + The full path to the project folder. + .PARAMETER containerName + The name of the build container to run the tests against. + .PARAMETER credential + The credential used to connect to the build container. + .PARAMETER installTestAppsJson + Path to a JSON file with the list of installed test apps. + .PARAMETER runTestsOverride + Optional scriptblock overriding the built-in AlTool test runner (RunTestsInBcContainer). + #> + Param( + [hashtable] $settings, + [string] $projectPath, + [string] $containerName, + [System.Management.Automation.PSCredential] $credential, + [string] $installTestAppsJson = '', + [scriptblock] $runTestsOverride = $null + ) + + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installTestAppsJson + if (@($testApps).Count -eq 0) { + Write-Host "No test apps found to run tests in. Skipping test execution." + return + } + + Write-Host "Running tests against container '$containerName'" + + $testResultsFile = Join-Path $projectPath "TestResults.xml" + if (Test-Path $testResultsFile) { + Remove-Item $testResultsFile -Force + } + + # Test failures surface as warnings when treatTestFailuresAsWarnings is set, otherwise as errors. + $gitHubActionsSeverity = if ($settings.treatTestFailuresAsWarnings) { 'warning' } else { 'error' } + + $allTestsPassed = $true + Push-Location $projectPath + try { + foreach ($testApp in $testApps) { + $appJson = Get-AppJsonFromAppFile -appFile $testApp + Write-Host "Running tests in $($appJson.name) ($($appJson.id))" + + $runTestsParams = @{ + "containerName" = $containerName + "credential" = $credential + "companyName" = $settings.companyName + "extensionId" = $appJson.id + "appName" = $appJson.name + "JUnitResultFileName" = $testResultsFile + "AppendToJUnitResultFile" = $true + "detailed" = $true + "GitHubActions" = $gitHubActionsSeverity + "returnTrueIfAllPassed" = $true + } + + if ($runTestsOverride) { + $passed = & $runTestsOverride -parameters $runTestsParams + } + else { + $passed = Invoke-AlToolTestRun -Parameters $runTestsParams + } + + if (-not $passed) { + $allTestsPassed = $false + } + } + } + finally { + Pop-Location + } + + if (-not $allTestsPassed) { + if ($settings.treatTestFailuresAsWarnings) { + OutputWarning -message "There are test failures, but they are treated as warnings (treatTestFailuresAsWarnings is set)." + } + else { + throw "There are test failures." + } + } +} + +Export-ModuleMember -Function Invoke-AlGoTestRun, Get-TestAppsToRun diff --git a/Actions/RunTests/action.yaml b/Actions/RunTests/action.yaml new file mode 100644 index 0000000000..4cd99644f4 --- /dev/null +++ b/Actions/RunTests/action.yaml @@ -0,0 +1,35 @@ +name: Run Tests +author: Microsoft Corporation +inputs: + shell: + description: Shell in which you want to run the action (powershell or pwsh) + required: false + default: powershell + token: + description: The GitHub token running the action + required: false + default: ${{ github.token }} + project: + description: Project folder + required: false + default: '.' + installTestAppsJson: + description: A path to a JSON-formatted list of test apps to run tests in + required: false + default: '' +runs: + using: composite + steps: + - name: run + shell: ${{ inputs.shell }} + env: + _token: ${{ inputs.token }} + _project: ${{ inputs.project }} + _installTestAppsJson: ${{ inputs.installTestAppsJson }} + run: | + ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "RunTests" -Action { + ${{ github.action_path }}/RunTests.ps1 -token $ENV:_token -project $ENV:_project -installTestAppsJson $ENV:_installTestAppsJson + } +branding: + icon: terminal + color: blue diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 310d6f6007..3bc9d2ec09 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,3 +1,12 @@ +### Separate test execution from RunPipeline (PREVIEW) + +A new `useSeparateTestAction` setting (default `false`) lets you move normal test execution (`testFolders`) out of the `RunPipeline` action and into a new dedicated `RunTests` action. When enabled, `RunPipeline` still compiles, publishes and installs the apps and keeps the build container alive, but does not run the normal tests. A new `RunTests` action then runs the tests against that same container and produces the same `TestResults.xml`. Only normal tests are affected; BCPT and page scripting tests are still executed by `RunPipeline`. When the setting is `false`, behavior is unchanged. + +By default, the `RunTests` action runs the tests through Microsoft's headless `al runtests` (AlTool) runner instead of BcContainerHelper. It resolves the kept-alive container's connection settings, installs the AL developer tools (`dotnet tool install --prerelease`), runs each test codeunit in its own `al runtests` invocation and emits the same `TestResults.xml` schema, so downstream test result analysis is unchanged. To run the tests through BcContainerHelper (or any other runner) instead, supply a `RunTestsInBcContainer` override script; it fully replaces the built-in AlTool runner. + +> [!NOTE] +> The built-in AlTool runner does not run `Legacy` test-type codeunits or tests that require UI or client-callback interaction. Projects that rely on those should supply a `RunTestsInBcContainer` override script to run their tests through BcContainerHelper instead. + ### New `doNotPerformUpgrade` setting AL-Go now supports a new `doNotPerformUpgrade` setting that is passed through to `Run-AlPipeline`. Use it to skip the upgrade phase while still running the rest of the pipeline. diff --git a/Scenarios/settings.md b/Scenarios/settings.md index c0118f92d4..f24d63bcf1 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -244,6 +244,7 @@ Please read the release notes carefully when installing new versions of AL-Go fo | doNotBuildTests | This setting forces the pipeline to NOT build and run the tests and performance tests in testFolders and bcptTestFolders | false | | doNotRunTests | This setting forces the pipeline to NOT run the tests in testFolders. Tests are still being built and published. Note this setting can be set in a [workflow specific settings file](#where-are-the-settings-located) to only apply to that workflow | false | | doNotRunBcptTests | This setting forces the pipeline to NOT run the performance tests in testFolders. Performance tests are still being built and published. Note this setting can be set in a [workflow specific settings file](#where-are-the-settings-located) to only apply to that workflow | false | +| useSeparateTestAction | PREVIEW: When set to true, normal tests (testFolders) are no longer executed inside the RunPipeline action. Instead, RunPipeline still compiles, publishes and installs the apps and keeps the build container alive, and a separate RunTests action executes the tests afterwards against that container. This only affects normal tests; BCPT and page scripting tests are still executed by RunPipeline. Existing behavior is unchanged when this setting is false. | false | | memoryLimit | Specifies the memory limit for the build container. By default, this is left to BcContainerHelper to handle and will currently be set to 8G | 8G | | BcContainerHelperVersion | This setting can be set to a specific version (ex. 3.0.8) of BcContainerHelper to force AL-Go to use this version. **latest** means that AL-Go will use the latest released version. **preview** means that AL-Go will use the latest preview version. **dev** means that AL-Go will use the dev branch of containerhelper. | latest (or preview for AL-Go preview) | | unusedALGoSystemFiles (**deprecated**) | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | [ ] | diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index bc3ef290ea..21078c344c 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -109,7 +109,7 @@ jobs: shell: ${{ inputs.shell }} project: ${{ inputs.project }} buildMode: ${{ inputs.buildMode }} - get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotRunBcptTests,doNotRunpageScriptingTests,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade + get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotPublishApps,doNotRunBcptTests,doNotRunpageScriptingTests,useSeparateTestAction,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade - name: Run Build Initialize hook if: hashFiles(format('{0}/.AL-Go/BuildInitialize.ps1', inputs.project)) != '' @@ -229,6 +229,17 @@ jobs: baselineWorkflowSHA: ${{ inputs.baselineWorkflowSHA }} previousAppsPath: ${{ steps.DownloadPreviousRelease.outputs.PreviousAppsPath }} + - name: Run Tests + uses: microsoft/AL-Go-Actions/RunTests@main + if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && env.useSeparateTestAction == 'True' && env.doNotPublishApps == 'False' && env.doNotRunTests == 'False' + env: + Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' + BuildMode: ${{ inputs.buildMode }} + with: + shell: ${{ inputs.shell }} + project: ${{ inputs.project }} + installTestAppsJson: ${{ steps.DownloadProjectDependencies.outputs.DownloadedTestApps }} + - name: Sign id: sign if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && inputs.signArtifacts && env.doNotSignApps == 'False' && (env.keyVaultCodesignCertificateName != '' || (fromJson(env.trustedSigning).Endpoint != '' && fromJson(env.trustedSigning).Account != '' && fromJson(env.trustedSigning).CertificateProfile != '')) && (hashFiles(format('{0}/.buildartifacts/Apps/*.app',inputs.project)) != '') diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index bc3ef290ea..21078c344c 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -109,7 +109,7 @@ jobs: shell: ${{ inputs.shell }} project: ${{ inputs.project }} buildMode: ${{ inputs.buildMode }} - get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotRunBcptTests,doNotRunpageScriptingTests,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade + get: useCompilerFolder,workspaceCompilation,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,doNotPublishApps,doNotRunBcptTests,doNotRunpageScriptingTests,useSeparateTestAction,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules,trackALAlertsInGitHub,skipUpgrade - name: Run Build Initialize hook if: hashFiles(format('{0}/.AL-Go/BuildInitialize.ps1', inputs.project)) != '' @@ -229,6 +229,17 @@ jobs: baselineWorkflowSHA: ${{ inputs.baselineWorkflowSHA }} previousAppsPath: ${{ steps.DownloadPreviousRelease.outputs.PreviousAppsPath }} + - name: Run Tests + uses: microsoft/AL-Go-Actions/RunTests@main + if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && env.useSeparateTestAction == 'True' && env.doNotPublishApps == 'False' && env.doNotRunTests == 'False' + env: + Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' + BuildMode: ${{ inputs.buildMode }} + with: + shell: ${{ inputs.shell }} + project: ${{ inputs.project }} + installTestAppsJson: ${{ steps.DownloadProjectDependencies.outputs.DownloadedTestApps }} + - name: Sign id: sign if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && inputs.signArtifacts && env.doNotSignApps == 'False' && (env.keyVaultCodesignCertificateName != '' || (fromJson(env.trustedSigning).Endpoint != '' && fromJson(env.trustedSigning).Account != '' && fromJson(env.trustedSigning).CertificateProfile != '')) && (hashFiles(format('{0}/.buildartifacts/Apps/*.app',inputs.project)) != '') diff --git a/Tests/AlToolTestRunner.Test.ps1 b/Tests/AlToolTestRunner.Test.ps1 new file mode 100644 index 0000000000..5cf98dd63b --- /dev/null +++ b/Tests/AlToolTestRunner.Test.ps1 @@ -0,0 +1,213 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'Mock/callback parameters must match function signatures')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'Test-only credential')] +param() + +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 + +Import-Module (Join-Path $PSScriptRoot '../Actions/RunTests/AlToolTestRunner.psm1' -Resolve) -DisableNameChecking -Force + +Describe 'AlToolTestRunner.psm1 Tests' { + + BeforeAll { + # Re-import in the run phase so the module functions are guaranteed to be available even when + # this file runs in the same Invoke-Pester session as RunTests.Test.ps1. RunTests.psm1 + # imports AlToolTestRunner.psm1 as a nested module with -Force, which removes the standalone + # module's functions from the global scope during discovery. + Import-Module (Join-Path $PSScriptRoot '../Actions/RunTests/AlToolTestRunner.psm1' -Resolve) -DisableNameChecking -Force + } + + Context 'ConvertFrom-AlRunTestsOutput' { + It 'Parses pass and fail results, dropping phantom OnRun and aggregate entries' { + $lines = @( + 'Test run completed: 3 tests, 1 failed', + 'Results:', + 'PASS OnRun (5ms)', + 'PASS MyPassingTest (12ms)', + 'FAIL MyFailingTest (8ms)', + ' Assert.AreEqual failed. Expected 1, got 2.', + 'AL Callstack:', + ' "My Codeunit"(CodeUnit 130001).MyFailingTest line 3', + 'PASS (25ms)' + ) + + $results = ConvertFrom-AlRunTestsOutput -OutputLines $lines + + $results.Keys.Count | Should -Be 2 + $results.ContainsKey('OnRun') | Should -BeFalse + $results['MyPassingTest'].Outcome | Should -Be 'Pass' + $results['MyPassingTest'].Ms | Should -Be 12 + $results['MyFailingTest'].Outcome | Should -Be 'Fail' + $results['MyFailingTest'].Message | Should -Match 'Assert.AreEqual failed' + $results['MyFailingTest'].Stacktrace | Should -Match 'MyFailingTest line 3' + } + + It 'Returns an empty map when there is no Results block' { + $results = ConvertFrom-AlRunTestsOutput -OutputLines @('Some unrelated output', 'No results here') + $results.Keys.Count | Should -Be 0 + } + + It 'Parses skipped results' { + $lines = @('Results:', 'SKIP MySkippedTest (0ms)') + $results = ConvertFrom-AlRunTestsOutput -OutputLines $lines + $results['MySkippedTest'].Outcome | Should -Be 'Skip' + } + } + + Context 'Get-DisabledTestKeySet' { + It 'Builds per-method keys and whole-codeunit sets from a wildcard' { + $disabled = @( + [PSCustomObject]@{ codeunitName = 'My Tests'; method = 'TestOne' }, + [PSCustomObject]@{ codeunitName = 'Whole CU'; method = '*' } + ) + + $lookup = Get-DisabledTestKeySet -DisabledTests $disabled + + $lookup.Methods.ContainsKey('my tests::testone') | Should -BeTrue + $lookup.Codeunits.ContainsKey('whole cu') | Should -BeTrue + $lookup.Methods.ContainsKey('whole cu::*') | Should -BeFalse + } + + It 'Returns empty sets for an empty list' { + $lookup = Get-DisabledTestKeySet -DisabledTests @() + $lookup.Methods.Count | Should -Be 0 + $lookup.Codeunits.Count | Should -Be 0 + } + } + + Context 'Get-AlToolTestCodeunits' { + It 'Enumerates codeunits and filters disabled methods and whole codeunits' { + Mock -ModuleName AlToolTestRunner Get-TestsFromBcContainer { + @( + [PSCustomObject]@{ Id = 130001; Name = 'My Tests'; Tests = @('TestOne', 'TestTwo') }, + [PSCustomObject]@{ Id = 130002; Name = 'Whole CU'; Tests = @('X', 'Y') } + ) + } + $params = @{ + containerName = 'test' + credential = (New-Object System.Management.Automation.PSCredential('admin', (ConvertTo-SecureString 'password' -AsPlainText -Force))) + extensionId = [Guid]::NewGuid().ToString() + disabledTests = @( + [PSCustomObject]@{ codeunitName = 'My Tests'; method = 'TestTwo' }, + [PSCustomObject]@{ codeunitName = 'Whole CU'; method = '*' } + ) + } + + $codeunits = @(Get-AlToolTestCodeunits -Parameters $params) + + $codeunits.Count | Should -Be 1 + $codeunits[0].Name | Should -Be 'My Tests' + @($codeunits[0].Tests).Count | Should -Be 1 + $codeunits[0].Tests[0] | Should -Be 'TestOne' + } + + It 'Returns every codeunit when there are no disabled tests' { + Mock -ModuleName AlToolTestRunner Get-TestsFromBcContainer { + @([PSCustomObject]@{ Id = 130001; Name = 'My Tests'; Tests = @('TestOne') }) + } + $params = @{ + containerName = 'test' + credential = (New-Object System.Management.Automation.PSCredential('admin', (ConvertTo-SecureString 'password' -AsPlainText -Force))) + extensionId = [Guid]::NewGuid().ToString() + } + + $codeunits = @(Get-AlToolTestCodeunits -Parameters $params) + $codeunits.Count | Should -Be 1 + } + } + + Context 'Get-AlToolConnection' { + It 'Reads server instance and developer services port from the container configuration' { + Mock -ModuleName AlToolTestRunner Get-BcContainerServerConfiguration { + [PSCustomObject]@{ ServerInstance = 'MyBC'; DeveloperServicesPort = 7145 } + } + + $connection = Get-AlToolConnection -ContainerName 'mycontainer' + + $connection.Server | Should -Be 'http://mycontainer' + $connection.ServerInstance | Should -Be 'MyBC' + $connection.Port | Should -Be 7145 + } + + It 'Falls back to conventional defaults when configuration cannot be read' { + Mock -ModuleName AlToolTestRunner Get-BcContainerServerConfiguration { throw 'no such container' } + + $connection = Get-AlToolConnection -ContainerName 'mycontainer' + + $connection.Server | Should -Be 'http://mycontainer' + $connection.ServerInstance | Should -Be 'BC' + $connection.Port | Should -Be 7049 + } + } + + Context 'Get-AlToolCompany' { + It 'Honors an explicitly requested company name without querying the container' { + Mock -ModuleName AlToolTestRunner Get-CompanyInBcContainer { throw 'should not be called' } + $company = Get-AlToolCompany -ContainerName 'test' -Tenant 'default' -CompanyName 'CRONUS' + $company | Should -Be 'CRONUS' + } + + It 'Falls back to the container default company, preferring an evaluation company' { + Mock -ModuleName AlToolTestRunner Get-CompanyInBcContainer { + @( + [PSCustomObject]@{ companyName = 'CRONUS Real'; evaluationCompany = $false }, + [PSCustomObject]@{ companyName = 'CRONUS Eval'; evaluationCompany = $true } + ) + } + $company = Get-AlToolCompany -ContainerName 'test' -Tenant 'default' + $company | Should -Be 'CRONUS Eval' + } + } + + Context 'Add-JUnitTestSuite' { + BeforeEach { + $script:doc = New-Object System.Xml.XmlDocument + $script:doc.AppendChild($script:doc.CreateXmlDeclaration("1.0", "UTF-8", $null)) | Out-Null + $script:suites = $script:doc.CreateElement("testsuites") + $script:doc.AppendChild($script:suites) | Out-Null + } + + It 'Writes a passing suite with correct counts and no failure nodes' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + $methodResults = @{ TestA = @{ Outcome = 'Pass'; Ms = 10; Message = ''; Stacktrace = '' } } + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('TestA') -MethodResults $methodResults -ExtensionId 'ext-id' -AppName 'MyApp' ` + -Hostname 'host' -ElapsedSec 1.5 + + $failed | Should -Be 0 + $suite = $script:suites.SelectSingleNode('testsuite') + $suite.GetAttribute('name') | Should -Be '130001 My Tests' + $suite.GetAttribute('tests') | Should -Be '1' + $suite.GetAttribute('failures') | Should -Be '0' + $suite.SelectNodes('testcase/failure').Count | Should -Be 0 + } + + It 'Marks a requested method with no result as a failure' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('MissingTest') -MethodResults @{} -ExtensionId 'ext-id' -AppName 'MyApp' ` + -Hostname 'host' -ElapsedSec 0.0 + + $failed | Should -Be 1 + $suite = $script:suites.SelectSingleNode('testsuite') + $suite.GetAttribute('failures') | Should -Be '1' + $suite.SelectSingleNode('testcase/failure').GetAttribute('message') | Should -Match 'No result produced' + } + + It 'Records a failing method with its message and stacktrace' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + $methodResults = @{ TestA = @{ Outcome = 'Fail'; Ms = 4; Message = 'boom'; Stacktrace = 'line1;line2' } } + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('TestA') -MethodResults $methodResults -ExtensionId 'ext-id' -AppName 'MyApp' ` + -Hostname 'host' -ElapsedSec 0.5 + + $failed | Should -Be 1 + $failureNode = $script:suites.SelectSingleNode('testsuite/testcase/failure') + $failureNode.GetAttribute('message') | Should -Be 'boom' + $failureNode.InnerText | Should -Match 'line1' + $failureNode.InnerText | Should -Match 'line2' + } + } +} diff --git a/Tests/RunTests.Action.Test.ps1 b/Tests/RunTests.Action.Test.ps1 new file mode 100644 index 0000000000..e9bdf19523 --- /dev/null +++ b/Tests/RunTests.Action.Test.ps1 @@ -0,0 +1,28 @@ +Get-Module TestActionsHelper | Remove-Module -Force +Import-Module (Join-Path $PSScriptRoot 'TestActionsHelper.psm1') +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 + +Describe "RunTests Action Tests" { + BeforeAll { + $actionName = "RunTests" + $scriptRoot = Join-Path $PSScriptRoot "..\Actions\$actionName" -Resolve + $scriptName = "$actionName.ps1" + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'scriptPath', Justification = 'False positive.')] + $scriptPath = Join-Path $scriptRoot $scriptName + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'actionScript', Justification = 'False positive.')] + $actionScript = GetActionScript -scriptRoot $scriptRoot -scriptName $scriptName + } + + It 'Compile Action' { + Invoke-Expression $actionScript + } + + It 'Test action.yaml matches script' { + $outputs = [ordered]@{ + } + YamlTest -scriptRoot $scriptRoot -actionName $actionName -actionScript $actionScript -outputs $outputs + } + + # Call action + +} diff --git a/Tests/RunTests.Test.ps1 b/Tests/RunTests.Test.ps1 new file mode 100644 index 0000000000..318c66a804 --- /dev/null +++ b/Tests/RunTests.Test.ps1 @@ -0,0 +1,268 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'Mock/callback parameters must match function signatures')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'Test-only credential')] +param() + +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 + +. (Join-Path -Path $PSScriptRoot -ChildPath "../Actions/AL-Go-Helper.ps1" -Resolve) + +# Stub for the BcContainerHelper function so it can be mocked within the module scope +function Get-AppJsonFromAppFile { param($appFile) } + +Import-Module (Join-Path $PSScriptRoot '../Actions/RunTests/RunTests.psm1' -Resolve) -DisableNameChecking -Force + +Describe 'RunTests.psm1 Tests' { + BeforeAll { + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'testCredential', Justification = 'Used in tests')] + $testCredential = New-Object System.Management.Automation.PSCredential("admin", (ConvertTo-SecureString "password" -AsPlainText -Force)) + + function New-TestProject { + Param( + [string[]] $CompiledTestApps = @() + ) + $projectPath = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + $testAppsFolder = Join-Path $projectPath ".buildartifacts\TestApps" + New-Item -Path $testAppsFolder -ItemType Directory -Force | Out-Null + foreach ($app in $CompiledTestApps) { + New-Item -Path (Join-Path $testAppsFolder $app) -ItemType File -Force | Out-Null + } + return $projectPath + } + } + + Context 'Get-TestAppsToRun' { + It 'Collects compiled test apps from the build artifacts folder' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app', 'App2.Test.app') + $settings = @{ runTestsInAllInstalledTestApps = $false } + + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath + + @($testApps).Count | Should -Be 2 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Includes installed test apps (unwrapping parentheses) when runTestsInAllInstalledTestApps is set' { + $projectPath = New-TestProject + $installedApp1 = Join-Path $projectPath 'Installed1.app' + $installedApp2 = Join-Path $projectPath 'Installed2.app' + New-Item -Path $installedApp1 -ItemType File -Force | Out-Null + New-Item -Path $installedApp2 -ItemType File -Force | Out-Null + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @($installedApp1, "($installedApp2)") | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ runTestsInAllInstalledTestApps = $true } + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson + + @($testApps).Count | Should -Be 2 + $testApps | Should -Contain $installedApp1 + $testApps | Should -Contain $installedApp2 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Ignores installed test apps when runTestsInAllInstalledTestApps is not set' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $installedApp = Join-Path $projectPath 'Installed1.app' + New-Item -Path $installedApp -ItemType File -Force | Out-Null + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @($installedApp) | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ runTestsInAllInstalledTestApps = $false } + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson + + @($testApps).Count | Should -Be 1 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Does not throw and returns compiled test apps when installTestAppsJson is an empty array' { + # Regression: in Windows PowerShell 5.1 ConvertFrom-Json emits a JSON array as a single + # object, so an empty '[]' previously surfaced as a one-element System.Object[] and threw + # "does not contain a method named 'TrimStart'". A test project with no installed test apps + # (the common case) must still return its compiled test apps without throwing. + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @() | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ runTestsInAllInstalledTestApps = $true } + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson + + @($testApps).Count | Should -Be 1 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Includes a single installed test app when runTestsInAllInstalledTestApps is set' { + # Regression: a single-element JSON array must enumerate to the string element (not the + # whole array) on both Windows PowerShell 5.1 and PowerShell 7. + $projectPath = New-TestProject + $installedApp = Join-Path $projectPath 'Installed1.app' + New-Item -Path $installedApp -ItemType File -Force | Out-Null + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @("($installedApp)") | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ runTestsInAllInstalledTestApps = $true } + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson + + @($testApps).Count | Should -Be 1 + $testApps | Should -Contain $installedApp + Remove-Item -Path $projectPath -Recurse -Force + } + } + + Context 'Invoke-AlGoTestRun' { + It 'Does not run tests when there are no test apps' { + $projectPath = New-TestProject + $script:runnerCalls = 0 + $override = { param($parameters) $script:runnerCalls++; return $true } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:runnerCalls | Should -Be 0 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Runs tests in every test app when tests pass' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app', 'App2.Test.app') + $script:runnerCalls = 0 + $override = { param($parameters) $script:runnerCalls++; return $true } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | Should -Not -Throw + + $script:runnerCalls | Should -Be 2 + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Throws when a test fails and treatTestFailuresAsWarnings is not set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $override = { param($parameters) return $false } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | Should -Throw + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Does not throw when a test fails but treatTestFailuresAsWarnings is set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $override = { param($parameters) return $false } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $true } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | Should -Not -Throw + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Passes GitHubActions severity error when treatTestFailuresAsWarnings is not set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:capturedSeverity = $null + $override = { param($parameters) $script:capturedSeverity = $parameters.GitHubActions; return $true } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:capturedSeverity | Should -Be 'error' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Passes GitHubActions severity warning when treatTestFailuresAsWarnings is set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:capturedSeverity = $null + $override = { param($parameters) $script:capturedSeverity = $parameters.GitHubActions; return $true } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $true } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:capturedSeverity | Should -Be 'warning' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Builds a parameter set that is valid for the real Run-TestsInBcContainer cmdlet' { + # Guard against parameter drift: every key/value passed to the BcContainerHelper test + # runner is validated against the real cmdlet signature (parameter names and ValidateSet + # values). This catches invalid parameter names and out-of-set values locally instead of + # only surfacing them in CI, where the real cmdlet is actually invoked. + $command = Get-Command -Name 'Run-TestsInBcContainer' -ErrorAction SilentlyContinue + if (-not $command) { + Set-ItResult -Skipped -Because 'BcContainerHelper (Run-TestsInBcContainer) is not available in this environment' + return + } + if (($command -is [System.Management.Automation.AliasInfo]) -and $command.ResolvedCommand) { $command = $command.ResolvedCommand } + + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:capturedParams = $null + $override = { param($parameters) $script:capturedParams = $parameters; return $true } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:capturedParams | Should -Not -BeNullOrEmpty + foreach ($key in $script:capturedParams.Keys) { + $parameter = $command.Parameters[$key] + $parameter | Should -Not -BeNullOrEmpty -Because "'$key' must be a real parameter of Run-TestsInBcContainer" + + $validateSet = $parameter.Attributes | Where-Object { $_ -is [System.Management.Automation.ValidateSetAttribute] } | Select-Object -First 1 + if ($validateSet) { + $validateSet.ValidValues | Should -Contain $script:capturedParams[$key] -Because "the value for '$key' must be one of its allowed ValidateSet values" + } + } + + Remove-Item -Path $projectPath -Recurse -Force + } + } + + Context 'Invoke-AlGoTestRun (default AlTool runner)' { + It 'Runs the AlTool runner for every test app when no override is supplied' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + Mock -ModuleName RunTests Invoke-AlToolTestRun { return $true } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app', 'App2.Test.app') + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential } | Should -Not -Throw + + Should -Invoke -ModuleName RunTests Invoke-AlToolTestRun -Times 2 -Exactly + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Passes the container and credential through to the AlTool runner' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + Mock -ModuleName RunTests Invoke-AlToolTestRun { return $true } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = 'CRONUS'; treatTestFailuresAsWarnings = $false } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'mycontainer' -credential $testCredential + + Should -Invoke -ModuleName RunTests Invoke-AlToolTestRun -Times 1 -Exactly -ParameterFilter { + $Parameters.containerName -eq 'mycontainer' -and $Parameters.companyName -eq 'CRONUS' -and ($Parameters.credential -is [System.Management.Automation.PSCredential]) + } + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Throws when the AlTool runner reports failure and treatTestFailuresAsWarnings is not set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + Mock -ModuleName RunTests Invoke-AlToolTestRun { return $false } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential } | Should -Throw + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Does not throw when the AlTool runner reports failure but treatTestFailuresAsWarnings is set' { + Mock -ModuleName RunTests Get-AppJsonFromAppFile { [PSCustomObject]@{ id = [Guid]::NewGuid().ToString(); name = 'TestApp' } } + Mock -ModuleName RunTests Invoke-AlToolTestRun { return $false } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $true } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential } | Should -Not -Throw + + Remove-Item -Path $projectPath -Recurse -Force + } + } +}