--- a/src/Futhark/CLI/Test.hs
+++ b/src/Futhark/CLI/Test.hs
@@ -5,6 +5,7 @@
 
 import Control.Applicative.Lift (Errors, Lift (..), failure, runErrors)
 import Control.Concurrent
+import Control.Concurrent.Async
 import Control.Exception
 import Control.Monad
 import Control.Monad.Except (ExceptT (..), MonadError, runExceptT, withExceptT)
@@ -105,7 +106,14 @@
   context prog_ctx $
     pureTestResults $
       liftIO $
-        withServer (futharkServerCfg to_run to_run_args) f
+        withServer (futharkServerCfg to_run to_run_args) $ \server ->
+          race (threadDelay $ 5 * 60 * 1000000) (f server) >>= \case
+            Left _ -> do
+              abortServer server
+              -- This value won't be used since abortServer will
+              -- already cause a non-zero status for wait.
+              pure undefined
+            Right r -> pure r
 
 data TestMode
   = -- | Only type check.
--- a/futhark.cabal
+++ b/futhark.cabal
@@ -433,6 +433,7 @@
       aeson >=2.0.0.0
     , ansi-terminal >=0.6.3.1
     , array >=0.4
+    , async >=2.2
     , base >=4.15 && <5
     , base16-bytestring
     , binary >=0.8.3
@@ -451,7 +452,7 @@
     , filepath >=1.4.1.1
     , free >=5.1.10
     , futhark-data >= 1.1.1.0
-    , futhark-server >= 1.2.2.1
+    , futhark-server >= 1.2.3.0
     , futhark-manifest >= 1.5.0.0
     , githash >=0.1.6.1
     , half >= 0.3
