diff --git a/sdk/UNRELEASED.md b/sdk/UNRELEASED.md index 6ded282c154b..1bcdeae4dc93 100644 --- a/sdk/UNRELEASED.md +++ b/sdk/UNRELEASED.md @@ -9,6 +9,25 @@ schedule, i.e. if you add an entry effective at or after the first header, prepend the new date header that corresponds to the Wednesday after your change. +## Until 2026-07-21 (Exclusive) + +### Improved daml test output +- Test summary now appears at the end of output, making pass/fail results + immediately visible +- Summary header shows pass/fail counts (e.g., "Test Summary: 2 failed, 3 + passed") +- When tests fail, only failed tests are shown in the summary to focus attention + on what needs fixing +- Colored output is auto-detected based on terminal capabilities +- Coverage statistics are now hidden by default; use --show-coverage to display + them +- Test file paths in the summary are now relative to the project root for + readability +- Tests are grouped by file, with cleaner formatting for multiple tests in the + same file +- Simplified diagnostic output: removed verbose LSP formatting, now shows just + the file path, script name, and error message + ## Until 2026-07-15 (Exclusive) - Daml Stdlib: add `(:|)` and `(<|)` synonyms for `DA.NonEmpty`, and define `Show` using the same representation (PR #22460) diff --git a/sdk/compiler/damlc/daml-compiler/src/DA/Daml/Compiler/Output.hs b/sdk/compiler/damlc/daml-compiler/src/DA/Daml/Compiler/Output.hs index f8a21b05d9c1..202480821df6 100644 --- a/sdk/compiler/damlc/daml-compiler/src/DA/Daml/Compiler/Output.hs +++ b/sdk/compiler/damlc/daml-compiler/src/DA/Daml/Compiler/Output.hs @@ -18,6 +18,7 @@ import Development.IDE.Core.Shake (NotificationHandler(..)) import Development.IDE.Types.Diagnostics import Development.IDE.Types.Location import qualified Language.LSP.Types as LSP +import System.Directory (makeAbsolute) import System.IO import Control.Exception (bracket) @@ -49,13 +50,20 @@ writeOutputBSL :: FilePath -> BSL.ByteString -> IO () writeOutputBSL = writeOutputWith BSL.hPutStr -- WARNING: Here be dragons --- T.putStrLn is locale-dependent. This seems to cause issues with Nix’ patched glibc that +-- T.putStrLn is locale-dependent. This seems to cause issues with Nix' patched glibc that -- relies on LOCALE_ARCHIVE being set correctly. This is the case in our dev-env -- but not when we ship the SDK. If LOCALE_ARCHIVE is not set properly the colored --- diagnostics get eaten somewhere in glibc and we don’t even get a write syscall containing them. +-- diagnostics get eaten somewhere in glibc and we don't even get a write syscall containing them. printDiagnostics :: Handle -> [FileDiagnostic] -> IO () printDiagnostics _ [] = return () -printDiagnostics handle xs = BS.hPutStrLn handle $ T.encodeUtf8 $ showDiagnosticsColored xs +printDiagnostics handle xs = do + xs' <- mapM makeAbsoluteDiag xs + BS.hPutStrLn handle $ T.encodeUtf8 $ showDiagnosticsColored xs' + +makeAbsoluteDiag :: FileDiagnostic -> IO FileDiagnostic +makeAbsoluteDiag (fp, showDiag, diag) = do + absPath <- makeAbsolute (fromNormalizedFilePath fp) + pure (toNormalizedFilePath' absPath, showDiag, diag) diagnosticsLogger :: NotificationHandler diagnosticsLogger = hDiagnosticsLogger stderr diff --git a/sdk/compiler/damlc/daml-ide-core/src/Development/IDE/Core/Rules/Daml.hs b/sdk/compiler/damlc/daml-ide-core/src/Development/IDE/Core/Rules/Daml.hs index 88c07474720e..d60db6c49d22 100644 --- a/sdk/compiler/damlc/daml-ide-core/src/Development/IDE/Core/Rules/Daml.hs +++ b/sdk/compiler/damlc/daml-ide-core/src/Development/IDE/Core/Rules/Daml.hs @@ -1028,7 +1028,7 @@ runSingleScriptRule = forM scripts $ \(script, loc) -> do (vr, res) <- runScript scriptService (Just file) ctxId (LF.moduleName m) script let range = maybe noRange sourceLocToRange loc - pure (toDiagnostics lvl world file range res, (vr, res)) + pure (toDiagnostics lvl world file range targetScriptName res, (vr, res)) let (diags, results) = unzip scriptResults pure (concat diags, Just results) @@ -1064,9 +1064,10 @@ toDiagnostics :: -> LF.World -> NormalizedFilePath -> Range + -> T.Text -- ^ Script name -> Either SS.Error SS.ScriptResult -> [FileDiagnostic] -toDiagnostics lvl world scriptFile scriptRange = \case +toDiagnostics lvl world scriptFile scriptRange scriptName = \case Left err -> pure $ mkDiagnostic DsError (scriptFile, scriptRange) $ formatScriptError lvl world err Right SS.ScriptResult{..} -> @@ -1079,7 +1080,7 @@ toDiagnostics lvl world scriptFile scriptRange = \case mkDiagnostic severity (file, range) pretty = (file, ShowDiag, ) $ Diagnostic { _range = range , _severity = Just severity - , _source = Just "Script" + , _source = Just $ "Script: " <> scriptName , _message = Pretty.renderPlain pretty , _code = Nothing , _tags = Nothing diff --git a/sdk/compiler/damlc/lib/DA/Cli/Damlc.hs b/sdk/compiler/damlc/lib/DA/Cli/Damlc.hs index 9d25fc962b1b..d8f6f5798de1 100644 --- a/sdk/compiler/damlc/lib/DA/Cli/Damlc.hs +++ b/sdk/compiler/damlc/lib/DA/Cli/Damlc.hs @@ -504,7 +504,7 @@ runTestsInPackageOrFiles packageLocationOpts Nothing allTests _ coverage color m -- Therefore we keep the behavior of only passing the root file -- if source points to a specific file. files <- getDamlRootFiles pSrc - execTest files allTests coverage color mbJUnitOutput (Just pkgConfig) cliOptions tableOutputPath transactionsOutputPath coveragePaths coverageFilters + execTest files allTests coverage color mbJUnitOutput (Just pkgConfig) cliOptions tableOutputPath transactionsOutputPath coveragePaths coverageFilters (Just pPath) runTestsInPackageOrFiles packageLocationOpts (Just inFiles) allTests _ coverage color mbJUnitOutput cliOptions initPkgDb tableOutputPath transactionsOutputPath coveragePaths coverageFilters = Command Test (Just packageLocationOpts) effect where effect = withPackageRoot (packageRoot packageLocationOpts) (packageLocationCheck packageLocationOpts) $ \mPackageRoot relativize -> do cliOptions <- addResolutionData cliOptions @@ -518,7 +518,7 @@ runTestsInPackageOrFiles packageLocationOpts (Just inFiles) allTests _ coverage Just packagePath -> withMaybeConfig (withPackageConfig (PackagePath packagePath)) pure Nothing -> pure Nothing inFiles' <- mapM (fmap toNormalizedFilePath' . relativize) inFiles - execTest inFiles' allTests coverage color mbJUnitOutput mPkgConfig cliOptions tableOutputPath transactionsOutputPath coveragePaths coverageFilters + execTest inFiles' allTests coverage color mbJUnitOutput mPkgConfig cliOptions tableOutputPath transactionsOutputPath coveragePaths coverageFilters mPackageRoot cmdInspect :: Mod CommandFields Command cmdInspect = diff --git a/sdk/compiler/damlc/lib/DA/Cli/Damlc/Test.hs b/sdk/compiler/damlc/lib/DA/Cli/Damlc/Test.hs index f4e216612b2d..00bc6079f54a 100644 --- a/sdk/compiler/damlc/lib/DA/Cli/Damlc/Test.hs +++ b/sdk/compiler/damlc/lib/DA/Cli/Damlc/Test.hs @@ -41,7 +41,6 @@ import qualified Data.Text as T import qualified Data.Text.IO as TIO import qualified Data.Text.Lazy as TL import Data.Tuple.Extra -import qualified Data.Vector as V import Development.IDE.Core.API import Development.IDE.Core.IdeState.Daml import Development.IDE.Core.RuleTypes.Daml @@ -52,11 +51,11 @@ import qualified Development.Shake as Shake import Safe import qualified ScriptService as SS import qualified DA.Cli.Damlc.Test.TestResults as TR -import System.Console.ANSI (SGR(..), setSGRCode, Underlining(..), ConsoleIntensity(..)) +import System.Console.ANSI (SGR(..), setSGRCode, Underlining(..), ConsoleIntensity(..), Color(..), ColorIntensity(..), ConsoleLayer(..)) import System.Directory (createDirectoryIfMissing) import System.Exit (exitFailure) import System.FilePath -import System.IO (hPutStrLn, stderr) +import System.IO (hIsTerminalDevice, hPutStrLn, stderr, stdout) import System.IO.Error (isPermissionError, isAlreadyExistsError, isDoesNotExistError) import qualified Text.XML.Light as XML import qualified Text.Blaze.Html.Renderer.Text as Blaze @@ -67,6 +66,8 @@ import ComponentVersion.Class (ComponentVersioned) newtype UseColor = UseColor {getUseColor :: Bool} newtype ShowCoverage = ShowCoverage {getShowCoverage :: Bool} + +type ScriptTestResult = (TR.LocalOrExternal, ScriptName, Either SSC.Error SSC.ScriptResult) newtype CoverageFilter = CoverageFilter {getCoverageFilter :: Regex} newtype RunAllOption = RunAllOption {getRunAllTests :: Bool} newtype TableOutputPath = TableOutputPath {getTableOutputPath :: Maybe String} @@ -91,16 +92,27 @@ execTest -> TransactionsOutputPath -> CoveragePaths -> [CoverageFilter] + -> Maybe FilePath -> IO () -execTest inFiles runAllOption coverage color mbJUnitOutput mPkgConfig opts tableOutputPath transactionsOutputPath resultsIO coverageFilters = do +execTest inFiles runAllOption coverage color mbJUnitOutput mPkgConfig opts tableOutputPath transactionsOutputPath resultsIO coverageFilters mbProjectPath = do loggerH <- getLogger opts "test" + color <- if getUseColor color then pure color else do + isTTY <- hIsTerminalDevice stdout + pure $ UseColor isTTY let optsWithPkg = case mPkgConfig of Just PackageConfigFields{..} -> opts { optMbPackageName = Just pName, optMbPackageVersion = pVersion } Nothing -> opts + -- Prefer package name over path for identification in headers + packageIdentifier = case mPkgConfig of + Just PackageConfigFields{..} -> Just $ T.unpack $ LF.unPackageName pName + Nothing -> mbProjectPath + printTestSuiteBegin color packageIdentifier withDamlIdeState optsWithPkg loggerH noopLogger $ \h -> do flip finally (getDiagnostics h >>= printDiagnostics stderr) $ do - runAndReport h inFiles (optDetailLevel opts) (optDamlLfVersion opts) runAllOption coverage color mbJUnitOutput tableOutputPath transactionsOutputPath resultsIO coverageFilters + summaryResults <- runAndReport h inFiles (optDetailLevel opts) (optDamlLfVersion opts) runAllOption coverage mbJUnitOutput tableOutputPath transactionsOutputPath resultsIO coverageFilters diags <- getDiagnostics h + -- Pass both identifier (for header) and path (for relativizing test file paths) + printSummary color packageIdentifier mbProjectPath summaryResults when (any (\(_, _, diag) -> Just DsError == _severity diag) diags) exitFailure loadAggregatePrintResults :: CoveragePaths -> [CoverageFilter] -> ShowCoverage -> Maybe TR.TestResults -> IO () @@ -131,19 +143,17 @@ runAndReport :: -> LF.Version -> RunAllOption -> ShowCoverage - -> UseColor -> Maybe FilePath -> TableOutputPath -> TransactionsOutputPath -> CoveragePaths -> [CoverageFilter] - -> IO () -runAndReport ideState inFiles lvl lfVersion runAllOption coverage color mbJUnitOutput tableOutputPath transactionsOutputPath resultsIO coverageFilters = do + -> IO [ScriptTestResult] +runAndReport ideState inFiles lvl lfVersion runAllOption coverage mbJUnitOutput tableOutputPath transactionsOutputPath resultsIO coverageFilters = do (localResults, extResults) <- runAllScripts ideState inFiles runAllOption let allResults = localResults ++ extResults let allPackages = [loe | TR.ScriptResults loe _ _ <- allResults] - -- print test summary after all tests have run - printSummary color [(loe, scriptName, res) | TR.ScriptResults loe _ (Just results) <- allResults, (scriptName, res) <- results] + let summaryResults = [(loe, scriptName, res) | TR.ScriptResults loe _ (Just results) <- allResults, (scriptName, res) <- results] let newTestResults = TR.scriptResultsToTestResults allPackages allResults loadAggregatePrintResults resultsIO coverageFilters coverage (Just newTestResults) @@ -179,6 +189,8 @@ runAndReport ideState inFiles lvl lfVersion runAllOption coverage color mbJUnitO ] writeFile junitOutput $ XML.showTopElement $ toJUnit res + pure summaryResults + runAllScripts :: IdeState -> [NormalizedFilePath] -> RunAllOption -> IO ([TR.ScriptResults], [TR.ScriptResults]) runAllScripts h inFiles (RunAllOption runAllOption) = do -- make sure none of the files disappear @@ -295,25 +307,98 @@ failedTestOutput h file = do pure $ map (, Just errMsg) scriptNames -printSummary :: UseColor -> [(TR.LocalOrExternal, ScriptName, Either SSC.Error SSC.ScriptResult)] -> IO () -printSummary color res = +printTestSuiteBegin :: UseColor -> Maybe String -> IO () +printTestSuiteBegin color mbIdentifier = + whenJust mbIdentifier $ \identifier -> do + let colored = getUseColor color + putStrLn $ + (if colored then setSGRCode [SetConsoleIntensity BoldIntensity] else "") + <> "Running tests (" <> identifier <> ") ..." + <> (if colored then setSGRCode [] else "") + +printSummary :: UseColor -> Maybe String -> Maybe FilePath -> [ScriptTestResult] -> IO () +printSummary color mbIdentifier mbProjectPath res = liftIO $ do - putStrLn $ - unlines - [ setSGRCode [SetUnderlining SingleUnderline, SetConsoleIntensity BoldIntensity] - , "Test Summary" <> setSGRCode [] - ] - printScriptResults color res - -printScriptResults :: UseColor -> [(TR.LocalOrExternal, ScriptName, Either SSC.Error SS.ScriptResult)] -> IO () -printScriptResults color results = do - liftIO $ forM_ results $ \(loe, ScriptName scriptName, resultOrErr) -> do - let name = DA.Pretty.pretty (TR.localOrExternalName loe) <> ":" <> DA.Pretty.pretty scriptName - let stringStyleToRender = if getUseColor color then DA.Pretty.renderColored else DA.Pretty.renderPlain - putStrLn $ stringStyleToRender $ - case resultOrErr of - Left _err -> name <> ": " <> DA.Pretty.error_ "failed" - Right result -> name <> ": " <> prettyResult result + let failedTests = [r | r@(_, _, Left _) <- res] + nFailed = length failedTests + nPassed = length res - nFailed + nTotal = length res + colored = getUseColor color + identifierSuffix = maybe "" (\ident -> " (" <> ident <> ")") mbIdentifier + + -- Handle the "no tests found" case + if nTotal == 0 + then do + putStrLn $ + (if colored then setSGRCode [SetUnderlining SingleUnderline, SetConsoleIntensity BoldIntensity] else "") + <> "Test Summary" <> identifierSuffix + <> (if colored then setSGRCode [] else "") + <> ": No tests found" + else do + let countLine + | nFailed > 0 = + (if colored then setSGRCode [SetColor Foreground Vivid Red] else "") + <> show nFailed <> " failed" + <> (if colored then setSGRCode [] else "") + <> ", " <> show nPassed <> " passed" + | otherwise = + (if colored then setSGRCode [SetColor Foreground Vivid Green] else "") + <> show nPassed <> " passed" + <> (if colored then setSGRCode [] else "") + + -- Print combined header line + putStrLn $ + (if colored then setSGRCode [SetUnderlining SingleUnderline, SetConsoleIntensity BoldIntensity] else "") + <> "Test Summary" <> identifierSuffix + <> (if colored then setSGRCode [] else "") + <> ": " <> countLine + + -- Only show failed tests in summary (passed tests are hidden when there are failures) + -- This keeps the output focused on what needs attention + when (nFailed == 0) $ + printScriptResults color mbProjectPath res + + -- Show failed tests last (most visible) + when (nFailed > 0) $ + printScriptResults color mbProjectPath failedTests + +printScriptResults :: UseColor -> Maybe FilePath -> [ScriptTestResult] -> IO () +printScriptResults color mbProjectPath results = do + -- Group results by file/package + let grouped = groupBy (\(loe1, _, _) (loe2, _, _) -> TR.localOrExternalName loe1 == TR.localOrExternalName loe2) results + liftIO $ forM_ grouped $ \groupResults -> do + -- Use short relative paths in test summary for readability. + -- Note: makeRelative returns the absolute path unchanged if paths have different + -- roots (e.g., different drives on Windows). This is acceptable as the absolute + -- path is still a valid, clickable path in the output. + let (loe, _, _) = head groupResults + loeName <- case loe of + TR.Local file _ -> do + let absPath = fromNormalizedFilePath file + let relativePath = maybe absPath (\projectPath -> makeRelative projectPath absPath) mbProjectPath + pure $ T.pack relativePath + TR.External _ -> pure $ TR.localOrExternalName loe + let colored = getUseColor color + failedResults = [(name, err) | (_, ScriptName name, Left err) <- groupResults] + passedResults = [(name, res) | (_, ScriptName name, Right res) <- groupResults] + nFailed = length failedResults + nPassed = length passedResults + -- Print summary line for this file + if nFailed > 0 + then do + let failedNames = map fst failedResults + testWord = if nFailed == 1 then "test" else "tests" + failedText = if colored then setSGRCode [SetColor Foreground Vivid Red] <> "failed" <> setSGRCode [] else "failed" + if nFailed == 1 + then putStrLn $ T.unpack loeName <> ": 1 " <> testWord <> " " <> failedText <> ": " <> T.unpack (head failedNames) + else do + putStrLn $ T.unpack loeName <> ": " <> show nFailed <> " " <> testWord <> " " <> failedText + forM_ failedNames $ \name -> + putStrLn $ " - " <> T.unpack name + else do + let testWord = if nPassed == 1 then "test" else "tests" + passedText = if colored then setSGRCode [SetColor Foreground Vivid Green] <> "passed" <> setSGRCode [] else "passed" + putStrLn $ T.unpack loeName <> ": " <> show nPassed <> " " <> testWord <> " " <> passedText prettyErr :: PrettyLevel -> LF.Version -> SSC.Error -> DA.Pretty.Doc Pretty.SyntaxClass @@ -328,15 +413,6 @@ prettyErr lvl lfVersion err = case err of SSC.ExceptionError e -> DA.Pretty.string $ show e -prettyResult :: SS.ScriptResult -> DA.Pretty.Doc Pretty.SyntaxClass -prettyResult result = - let nTx = length (SS.scriptResultScriptSteps result) - nActive = length $ filter (SS.isActive (SS.activeContractsFromScriptResult result)) (V.toList (SS.scriptResultNodes result)) - in DA.Pretty.typeDoc_ "ok, " - <> DA.Pretty.int nActive <> DA.Pretty.typeDoc_ " active contracts, " - <> DA.Pretty.int nTx <> DA.Pretty.typeDoc_ " transactions." - - toJUnit :: [(NormalizedFilePath, [(ScriptName, Maybe T.Text)])] -> XML.Element toJUnit results = XML.node diff --git a/sdk/compiler/damlc/lib/DA/Cli/Damlc/Test/TestResults.hs b/sdk/compiler/damlc/lib/DA/Cli/Damlc/Test/TestResults.hs index 803028ff6230..8a8a9c44e67e 100644 --- a/sdk/compiler/damlc/lib/DA/Cli/Damlc/Test/TestResults.hs +++ b/sdk/compiler/damlc/lib/DA/Cli/Damlc/Test/TestResults.hs @@ -23,6 +23,7 @@ import qualified Data.Text as T import qualified Data.Text.Lazy as TL import qualified Data.Set as S import qualified Data.Map.Strict as M +import Control.Monad (when) import Data.Maybe (mapMaybe) import Data.Foldable (fold) import Text.Printf @@ -537,6 +538,7 @@ printTestCoverage showCoverage testResults@TestResults { templates, interfaceIns [ printf " %s: %d" variety (M.size names) ] ++ [ " " ++ printer id | (id, _value) <- M.toList names ] in + when showCoverage $ putStrLn $ unlines $ concat diff --git a/sdk/compiler/damlc/tests/src/DamlcTest.hs b/sdk/compiler/damlc/tests/src/DamlcTest.hs index dabb3e8080d6..f4d93739ecbf 100644 --- a/sdk/compiler/damlc/tests/src/DamlcTest.hs +++ b/sdk/compiler/damlc/tests/src/DamlcTest.hs @@ -6,7 +6,7 @@ module DamlcTest {- HLINT ignore "locateRunfiles/package_app" -} -import Data.List.Extra (isInfixOf, isPrefixOf) +import Data.List.Extra (isInfixOf) import System.Directory import System.Environment.Blank import System.Exit @@ -268,27 +268,32 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ readProcessWithExitCode damlc [ "test" + , "--show-coverage" , "--package-root" , dir ] "" stderr @?= "" exitCode @?= ExitSuccess - let out = lines stdout + assertInfixOf "Running tests (test-coverage-report)" stdout + assertInfixOf "Test Summary (test-coverage-report)" stdout assertBool ("test coverage is reported correctly: " <> stdout) ( unlines [ "Modules internal to this package:" , "- Internal templates" , " 2 defined" , " 1 ( 50.0%) created" + , " internal templates never created: 1" + , " Foo:S" , "- Internal template choices" , " 3 defined" , " 1 ( 33.3%) exercised" + , " internal template choices never exercised: 2" + , " Foo:S:Archive" + , " Foo:T:Archive" ] `isInfixOf` stdout) - assertBool ("test summary is reported correctly: " <> out!!1) - ("Test Summary" `isPrefixOf` (out!!1)) - assertBool ("test summary is reported correctly: " <> out!!3) - ("./Foo.daml:x: ok, 0 active contracts, 2 transactions." == (out!!3)) + assertBool ("test result is reported correctly: " <> stdout) + ("./Foo.daml: 1 test passed" `isInfixOf` stdout) , testCase "Full test coverage report" $ do withTempDir $ \dir -> do writeFileUTF8 (dir "daml.yaml") $ unlines @@ -476,12 +481,9 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , bFilePath ] "" stderr @?= "" - assertBool ("Test coverage is reported correctly: " <> stdout) - (unlines - [ "B.daml:x: ok, 0 active contracts, 2 transactions." - , "a-0.0.1:testA: ok, 0 active contracts, 2 transactions." - ] `isInfixOf` - stdout) + assertBool ("Test results are reported correctly: " <> stdout) + ("B.daml: 1 test passed" `isInfixOf` stdout && + "a-0.0.1: 1 test passed" `isInfixOf` stdout) assertBool ("Internal module test coverage is reported correctly: " <> stdout) (unlines [ "Modules internal to this package:" @@ -600,12 +602,9 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , projDir "b" ] "" stderr @?= "" - assertBool ("Test coverage is reported correctly: " <> stdout) - (unlines - [ "./Mod2.daml:testMod1: ok, 0 active contracts, 4 transactions." - , "a-0.0.1:testMod1: ok, 0 active contracts, 4 transactions." - ] `isInfixOf` - stdout) + assertBool ("Test results are reported correctly: " <> stdout) + ("./Mod2.daml: 1 test passed" `isInfixOf` stdout && + "a-0.0.1: 1 test passed" `isInfixOf` stdout) assertBool ("Internal module test coverage is reported correctly: " <> stdout) (unlines [ "Modules internal to this package:" @@ -657,12 +656,9 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , projDir "b" ] "" stderr @?= "" - assertBool ("Exclude Deps: Test coverage is reported correctly: " <> stdout) - (unlines - [ "./Mod2.daml:testMod1: ok, 0 active contracts, 4 transactions." - , "a-0.0.1:testMod1: ok, 0 active contracts, 4 transactions." - ] `isInfixOf` - stdout) + assertBool ("Exclude Deps: Test results are reported correctly: " <> stdout) + ("./Mod2.daml: 1 test passed" `isInfixOf` stdout && + "a-0.0.1: 1 test passed" `isInfixOf` stdout) assertBool ("Exclude Deps: Internal module test coverage is reported correctly: " <> stdout) (unlines [ "Modules internal to this package:" @@ -710,12 +706,9 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , projDir "b" ] "" stderr @?= "" - assertBool ("Exclude Archive: Test coverage is reported correctly: " <> stdout) - (unlines - [ "./Mod2.daml:testMod1: ok, 0 active contracts, 4 transactions." - , "a-0.0.1:testMod1: ok, 0 active contracts, 4 transactions." - ] `isInfixOf` - stdout) + assertBool ("Exclude Archive: Test results are reported correctly: " <> stdout) + ("./Mod2.daml: 1 test passed" `isInfixOf` stdout && + "a-0.0.1: 1 test passed" `isInfixOf` stdout) assertBool ("Exclude Archive: Internal module test coverage is reported correctly: " <> stdout) (unlines [ "Modules internal to this package:" @@ -765,12 +758,9 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , projDir "b" ] "" stderr @?= "" - assertBool ("Exclude Archive and Dep: Test coverage is reported correctly: " <> stdout) - (unlines - [ "./Mod2.daml:testMod1: ok, 0 active contracts, 4 transactions." - , "a-0.0.1:testMod1: ok, 0 active contracts, 4 transactions." - ] `isInfixOf` - stdout) + assertBool ("Exclude Archive and Dep: Test results are reported correctly: " <> stdout) + ("./Mod2.daml: 1 test passed" `isInfixOf` stdout && + "a-0.0.1: 1 test passed" `isInfixOf` stdout) assertBool ("Exclude Archive and Dep: Internal module test coverage is reported correctly: " <> stdout) (unlines [ "Modules internal to this package:" @@ -846,23 +836,26 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , "--test-pattern" , "needle" , "--all" + , "--show-coverage" , "--package-root" , projDir "b" , "--files" , bFilePath ] "" stderr @?= "" + assertInfixOf "B.daml: 1 test passed" stdout + assertInfixOf "a-0.0.1: 1 test passed" stdout assertBool ("Test coverage is reported correctly: " <> stdout) (unlines - [ "B.daml:needleHaystack: ok, 0 active contracts, 0 transactions." - , "a-0.0.1:test_needleHaystack: ok, 0 active contracts, 0 transactions." - , "Modules internal to this package:" + [ "Modules internal to this package:" , "- Internal templates" , " 0 defined" , " 0 (100.0%) created" + , " internal templates never created: 0" , "- Internal template choices" , " 0 defined" , " 0 (100.0%) exercised" + , " internal template choices never exercised: 0" ] `isInfixOf` stdout) exitCode @?= ExitSuccess , testCase "Serialized results aggregate correctly" $ withTempDir $ \projDir -> do @@ -928,6 +921,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ readProcessWithExitCode damlc [ "test" + , "--show-coverage" , "--package-root", projDir , "-p", "testT1" , "--save-coverage", projDir "testT1-results" @@ -940,6 +934,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ readProcessWithExitCode damlc [ "test" + , "--show-coverage" , "--package-root", projDir , "-p", "testBoth" , "--save-coverage", projDir "testBoth-results" @@ -952,6 +947,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ readProcessWithExitCode damlc [ "test" + , "--show-coverage" , "--package-root", projDir , "--save-coverage", projDir "test-all-results" ] @@ -963,6 +959,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ readProcessWithExitCode damlc [ "test" + , "--show-coverage" , "--package-root", projDir , "--load-coverage-only" ] @@ -976,15 +973,18 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , "- Internal templates" , " 0 defined" , " 0 (100.0%) created" + , " internal templates never created: 0" , "- Internal template choices" , " 0 defined" , " 0 (100.0%) exercised" + , " internal template choices never exercised: 0" ] `isInfixOf` stdoutEmptyAggregate) (exitCode, stdoutAggregateTestT1, stderr) <- readProcessWithExitCode damlc [ "test" + , "--show-coverage" , "--package-root", projDir , "--load-coverage-only" , "--load-coverage", projDir "testT1-results" @@ -999,6 +999,8 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , "- Internal templates" , " 2 defined" , " 1 ( 50.0%) created" + , " internal templates never created: 1" + , " Main:T2" , "- Internal template choices" , " 6 defined" , " 2 ( 33.3%) exercised" @@ -1011,6 +1013,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ readProcessWithExitCode damlc [ "test" + , "--show-coverage" , "--package-root", projDir , "--load-coverage-only" , "--load-coverage", projDir "testBoth-results" @@ -1025,6 +1028,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , "- Internal templates" , " 2 defined" , " 2 (100.0%) created" + , " internal templates never created: 0" , "- Internal template choices" , " 6 defined" , " 2 ( 33.3%) exercised" @@ -1037,6 +1041,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ readProcessWithExitCode damlc [ "test" + , "--show-coverage" , "--package-root", projDir , "--load-coverage-only" , "--load-coverage", projDir "testT1-results" @@ -1052,6 +1057,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , "- Internal templates" , " 2 defined" , " 2 (100.0%) created" + , " internal templates never created: 0" , "- Internal template choices" , " 6 defined" , " 3 ( 50.0%) exercised" @@ -1064,6 +1070,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ readProcessWithExitCode damlc [ "test" + , "--show-coverage" , "--package-root", projDir , "--load-coverage-only" , "--load-coverage", projDir "testBoth-results" -- reorder the way in which we read the results, should be identical @@ -1080,6 +1087,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ readProcessWithExitCode damlc [ "test" + , "--show-coverage" , "--package-root", projDir , "--load-coverage", projDir "testT1-results" , "-p", "testBoth" @@ -1153,10 +1161,10 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , bFilePath] "" stderr @?= "" + assertInfixOf "B.daml: 1 test passed" stdout assertBool ("Test coverage is reported correctly: " <> stdout) (unlines - [ "B.daml:x: ok, 0 active contracts, 2 transactions." - , "Modules internal to this package:" + [ "Modules internal to this package:" , "- Internal templates" , " 2 defined" , " 1 ( 50.0%) created" @@ -1199,10 +1207,12 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ assertInfixOf "Script execution failed" stderr exitCode @?= ExitFailure 1 - let out = lines stdout - assertInfixOf "Test Summary" (out!!1) - assertInfixOf "Foo.daml:y: ok" (out!!3) - assertInfixOf "Foo.daml:x: failed" (out!!4) + assertInfixOf "Running tests (test-failing-script)" stdout + assertInfixOf "Test Summary (test-failing-script)" stdout + assertInfixOf "1 failed, 1 passed" stdout + assertBool ("passing test hidden when there are failures: " <> stdout) + (not ("Foo.daml: 1 test passed" `isInfixOf` stdout)) + assertInfixOf "Foo.daml: 1 test failed: x" stdout , testCase "damlc test --files outside of package" $ -- TODO: does this test make sense with a daml.yaml file? withTempDir $ \projDir -> do @@ -1229,7 +1239,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ , projDir "Main.daml" ] "" exitCode @?= ExitSuccess - assertBool ("Succeeding script in " <> stdout) ("Main.daml:test: ok" `isInfixOf` stdout) + assertBool ("Succeeding script in " <> stdout) ("Main.daml: 1 test passed" `isInfixOf` stdout) stderr @?= "" , testCase "damlc test --package-root relative" $ withTempDir $ \projDir -> do createDirectoryIfMissing True (projDir "relative") @@ -1351,8 +1361,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ "" stderr @?= "" exitCode @?= ExitSuccess - let out = lines stdout - out!!3 @?= "./Main.daml:test: ok, 1 active contracts, 3 transactions." + assertInfixOf "./Main.daml: 1 test passed" stdout ] <> [ testCase ("damlc test " <> unwords (args "") <> " in package") $ withTempDir $ \projDir -> do createDirectoryIfMissing True (projDir "a") @@ -1393,7 +1402,7 @@ testsForDamlcTest damlc scriptDar = testGroup "damlc test" $ : args projDir ) "" stderr @?= "" - assertBool ("Succeeding script in " <> stdout) ("B.daml:test: ok" `isInfixOf` stdout) + assertBool ("Succeeding script in " <> stdout) ("B.daml: 1 test passed" `isInfixOf` stdout) exitCode @?= ExitSuccess | args <- [\projDir -> ["--files", projDir "b" "B.daml"], const []] ]