From 5688820c3f47de147d6ddaa7098e7be56db753e4 Mon Sep 17 00:00:00 2001 From: Roger Bosman Date: Tue, 11 Aug 2026 16:19:44 +0200 Subject: [PATCH 1/2] Fix flaky lsp-tests timeout on m1 by reducing fold size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "scenario service does not interrupt on non-script messages" test uses an interpreted `foldl (+) 0 [1..N]` to create a long-running script, then sends a hover request during execution to verify the hover doesn't cancel the running script. The previous fold size of 10,000,000 took ~30s on a fast machine but could exceed the 60s timeout on loaded m1 CI runners — the only platform where this test flaked (see #20724). The fold just needs to run long enough for the hover to land during execution; it doesn't need to be maximally expensive. Changes: - Reduce fold size from 10M to 1M (~3s locally, ~6-10s under CI load) - Reduce timeout from 60s to 30s (1M should finish well within this) - Add timing instrumentation: measure time from hover-sent to script-finished, and assert it took at least 0.5s — proving the script was still running when the hover was processed, not that it finished before the hover arrived - Log elapsed time in test steps for future debugging run-all-tests: true Co-Authored-By: Claude Opus 4.6 Signed-off-by: Roger Bosman --- sdk/compiler/lsp-tests/BUILD.bazel | 1 + sdk/compiler/lsp-tests/src/Main.hs | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sdk/compiler/lsp-tests/BUILD.bazel b/sdk/compiler/lsp-tests/BUILD.bazel index 25de03a6410e..a75269501da2 100644 --- a/sdk/compiler/lsp-tests/BUILD.bazel +++ b/sdk/compiler/lsp-tests/BUILD.bazel @@ -39,6 +39,7 @@ da_haskell_test( "tasty", "tasty-hunit", "text", + "time", ], visibility = ["//visibility:public"], deps = [ diff --git a/sdk/compiler/lsp-tests/src/Main.hs b/sdk/compiler/lsp-tests/src/Main.hs index 1d791a75c34d..3b3009573fa5 100644 --- a/sdk/compiler/lsp-tests/src/Main.hs +++ b/sdk/compiler/lsp-tests/src/Main.hs @@ -8,6 +8,7 @@ module Main (main) where {- HLINT ignore "locateRunfiles/package_app" -} import Control.Concurrent +import Data.Time.Clock (getCurrentTime, diffUTCTime) import Control.Applicative.Combinators import Control.Lens hiding (List, children, (.=)) import Control.Monad @@ -745,8 +746,9 @@ scriptTests runScripts = testGroup "scripts" closeDoc script closeDoc main' - , localOption (mkTimeout 60000000) $ -- 60s timeout + , localOption (mkTimeout 30000000) $ -- 30s timeout testCaseSteps "scenario service does not interrupt on non-script messages" $ \step -> runScripts $ \_stderr -> do + let foldSize = 1000000 :: Integer -- open document with long-running script main' <- openDoc' "Main.daml" damlId $ T.unlines @@ -754,7 +756,7 @@ scriptTests runScripts = testGroup "scripts" , "module Main where" , "import Daml.Script" , "main : Script ()" - , "main = debug $ foldl (+) 0 [1..10000000]" + , "main = debug $ foldl (+) 0 [1.." <> T.pack (show foldSize) <> "]" ] liftIO $ step "Document opened." @@ -784,13 +786,21 @@ scriptTests runScripts = testGroup "scripts" -- run hover event _ <- sendRequest STextDocumentHover (HoverParams main' (Position 4 3) Nothing) + afterHoverTime <- liftIO getCurrentTime liftIO $ step "Hover sent..." -- Check that script did return and that log does not show any cancellations _changeResult <- waitForScriptDidChange + scriptDoneTime <- liftIO getCurrentTime + let afterHover = realToFrac (diffUTCTime scriptDoneTime afterHoverTime) :: Double + liftIO $ step $ "Script finished " ++ show afterHover ++ "s after hover was sent (fold size: " ++ show foldSize ++ ")" _scriptFinishedMessage <- liftIO $ assertUntilWithout _stderr "SCRIPT SERVICE STDOUT: Script finished." "SCRIPT SERVICE STDOUT: Script cancelled." liftIO $ step "Script returned without cancellation." + liftIO $ assertBool + ("Script finished too quickly after hover (" ++ show afterHover ++ "s) — fold size " ++ show foldSize ++ " may be too small to ensure the script is still running when hover is processed") + (afterHover >= 0.5) + closeDoc script closeDoc main' ] From 4faa0620667aac4855a743a134cbb89557d8ad94 Mon Sep 17 00:00:00 2001 From: Roger Bosman Date: Tue, 11 Aug 2026 17:00:55 +0200 Subject: [PATCH 2/2] Add timing guards to both script interrupt tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both "interrupts outdated script runs" and "does not interrupt on non-script messages" used a 10M-element fold. The "interrupts" test also flaked on linux arm and linux intel CI (not just m1) — the 30s timeout had to cover compilation + code lenses + script start + fold evaluation, and the 10M fold alone took 25-43s on loaded runners. Changes: - Reduce the "interrupts" test fold from 10M to 1M (same as "does not interrupt") - Add upper-bound timing guard to both tests: fail deterministically if total test time exceeds 50% of the timeout, with a message explaining the test is at risk of flaking and suggesting to reduce fold size or investigate slowness - The "does not interrupt" test keeps its existing lower-bound guard (afterHover >= 0.5s) proving the script was still running when the hover was processed On this machine, 1M fold gives total times of ~3s and ~5s against a 30s timeout — well within the 50% safety margin even under CI load. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Roger Bosman --- sdk/compiler/lsp-tests/src/Main.hs | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/sdk/compiler/lsp-tests/src/Main.hs b/sdk/compiler/lsp-tests/src/Main.hs index 3b3009573fa5..37854136c78d 100644 --- a/sdk/compiler/lsp-tests/src/Main.hs +++ b/sdk/compiler/lsp-tests/src/Main.hs @@ -687,9 +687,11 @@ scriptTests runScripts = testGroup "scripts" assertRegex (_vrcpContents changeResult) "Trace:[^/]+secondRun" closeDoc script closeDoc main' - , localOption (mkTimeout 30000000) $ -- 30s timeout + , let timeoutSeconds = 30 :: Int in + localOption (mkTimeout (fromIntegral timeoutSeconds * 1000000)) $ testCaseSteps "scenario service interrupts outdated script runs" $ \step -> runScripts $ \_stderr -> do - let mkDoc :: Integer -> T.Text + let foldSize = 1000000 :: Integer + mkDoc :: Integer -> T.Text mkDoc duration = T.unlines [ "{-# LANGUAGE ApplicativeDo #-}" , "module Main where" @@ -697,9 +699,10 @@ scriptTests runScripts = testGroup "scripts" , "main : Script ()" , "main = debug $ foldl (+) 0 [1.." <> T.pack (show duration) <> "]" ] + testStartTime <- liftIO getCurrentTime -- open document with long-running script - main' <- openDoc' "Main.daml" damlId $ mkDoc 10000000 + main' <- openDoc' "Main.daml" damlId $ mkDoc foldSize liftIO $ step "Document opened." -- wait until lenses processed, open script @@ -742,13 +745,22 @@ scriptTests runScripts = testGroup "scripts" -- check that returned value is new script _changeResult <- waitForScriptDidChange liftIO $ assertRegex (_vrcpContents _changeResult) "Trace:( |
)*276([^0-9]|$)" - liftIO $ step "Script results received." + testEndTime <- liftIO getCurrentTime + let totalTime = realToFrac (diffUTCTime testEndTime testStartTime) :: Double + maxSafeTime = fromIntegral timeoutSeconds * 0.5 + liftIO $ step $ "Script results received. Total time: " ++ show totalTime ++ "s (timeout: " ++ show timeoutSeconds ++ "s)" + + liftIO $ assertBool + ("Total test time " ++ show totalTime ++ "s is more than 50% of the " ++ show timeoutSeconds ++ "s timeout — this test is at risk of flaking under further CI load. Reduce fold size (currently " ++ show foldSize ++ ") or investigate why compilation/script evaluation is slow.") + (totalTime < maxSafeTime) closeDoc script closeDoc main' - , localOption (mkTimeout 30000000) $ -- 30s timeout + , let timeoutSeconds = 30 :: Int in + localOption (mkTimeout (fromIntegral timeoutSeconds * 1000000)) $ testCaseSteps "scenario service does not interrupt on non-script messages" $ \step -> runScripts $ \_stderr -> do let foldSize = 1000000 :: Integer + testStartTime <- liftIO getCurrentTime -- open document with long-running script main' <- openDoc' "Main.daml" damlId $ T.unlines @@ -797,10 +809,18 @@ scriptTests runScripts = testGroup "scripts" _scriptFinishedMessage <- liftIO $ assertUntilWithout _stderr "SCRIPT SERVICE STDOUT: Script finished." "SCRIPT SERVICE STDOUT: Script cancelled." liftIO $ step "Script returned without cancellation." + testEndTime <- liftIO getCurrentTime + let totalTime = realToFrac (diffUTCTime testEndTime testStartTime) :: Double + maxSafeTime = fromIntegral timeoutSeconds * 0.5 + liftIO $ assertBool ("Script finished too quickly after hover (" ++ show afterHover ++ "s) — fold size " ++ show foldSize ++ " may be too small to ensure the script is still running when hover is processed") (afterHover >= 0.5) + liftIO $ assertBool + ("Total test time " ++ show totalTime ++ "s is more than 50% of the " ++ show timeoutSeconds ++ "s timeout — this test is at risk of flaking under further CI load. Reduce fold size (currently " ++ show foldSize ++ ") or investigate why compilation/script evaluation is slow.") + (totalTime < maxSafeTime) + closeDoc script closeDoc main' ]