feat: add timeouts to fuzz testing

crates/config/src/fuzz.rs

@@ -32,6 +32,8 @@ pub struct FuzzConfig {
     pub failure_persist_file: Option<String>,
     /// show `console.log` in fuzz test, defaults to `false`
     pub show_logs: bool,
+    /// Optional timeout (in seconds) for each property test
+    pub timeout: Option<u32>,
 }
 
 impl Default for FuzzConfig {
@@ -45,6 +47,7 @@ impl Default for FuzzConfig {
             failure_persist_dir: None,
             failure_persist_file: None,
             show_logs: false,
+            timeout: None,
         }
     }
 }
@@ -61,6 +64,7 @@ impl FuzzConfig {
             failure_persist_dir: Some(cache_dir),
             failure_persist_file: Some("failures".to_string()),
             show_logs: false,
+            timeout: None,
         }
     }
 }
@@ -90,6 +94,7 @@ impl InlineConfigParser for FuzzConfig {
                 }
                 "failure-persist-file" => conf_clone.failure_persist_file = Some(value),
                 "show-logs" => conf_clone.show_logs = parse_config_bool(key, value)?,
+                "timeout" => conf_clone.timeout = Some(parse_config_u32(key, value)?),
                 _ => Err(InlineConfigParserError::InvalidConfigProperty(key))?,
             }
         }

crates/config/src/invariant.rs

@@ -36,6 +36,8 @@ pub struct InvariantConfig {
     pub failure_persist_dir: Option<PathBuf>,
     /// Whether to collect and display fuzzed selectors metrics.
     pub show_metrics: bool,
+    /// Optional timeout (in seconds) for each invariant test.
+    pub timeout: Option<u32>,
 }
 
 impl Default for InvariantConfig {
@@ -51,6 +53,7 @@ impl Default for InvariantConfig {
             gas_report_samples: 256,
             failure_persist_dir: None,
             show_metrics: false,
+            timeout: None,
         }
     }
 }
@@ -69,6 +72,7 @@ impl InvariantConfig {
             gas_report_samples: 256,
             failure_persist_dir: Some(cache_dir),
             show_metrics: false,
+            timeout: None,
         }
     }
 
@@ -108,6 +112,7 @@ impl InlineConfigParser for InvariantConfig {
                 }
                 "shrink-run-limit" => conf_clone.shrink_run_limit = parse_config_u32(key, value)?,
                 "show-metrics" => conf_clone.show_metrics = parse_config_bool(key, value)?,
+                "timeout" => conf_clone.timeout = Some(parse_config_u32(key, value)?),
                 _ => Err(InlineConfigParserError::InvalidConfigProperty(key.to_string()))?,
             }
         }

crates/evm/core/src/constants.rs

@@ -37,6 +37,9 @@ pub const MAGIC_ASSUME: &[u8] = b"FOUNDRY::ASSUME";
 /// Magic return value returned by the `skip` cheatcode. Optionally appended with a reason.
 pub const MAGIC_SKIP: &[u8] = b"FOUNDRY::SKIP";
 
+/// Test timeout return value.
+pub const TEST_TIMEOUT: &str = "FOUNDRY::TEST_TIMEOUT";
+
 /// The address that deploys the default CREATE2 deployer contract.
 pub const DEFAULT_CREATE2_DEPLOYER_DEPLOYER: Address =
     address!("3fAB184622Dc19b6109349B94811493BF2a45362");

crates/evm/evm/src/executors/fuzz/mod.rs

@@ -6,7 +6,7 @@ use eyre::Result;
 use foundry_common::evm::Breakpoints;
 use foundry_config::FuzzConfig;
 use foundry_evm_core::{
-    constants::MAGIC_ASSUME,
+    constants::{MAGIC_ASSUME, TEST_TIMEOUT},
     decode::{RevertDecoder, SkipReason},
 };
 use foundry_evm_coverage::HitMaps;
@@ -17,7 +17,11 @@ use foundry_evm_fuzz::{
 use foundry_evm_traces::SparsedTraceArena;
 use indicatif::ProgressBar;
 use proptest::test_runner::{TestCaseError, TestError, TestRunner};
-use std::{cell::RefCell, collections::BTreeMap};
+use std::{
+    cell::RefCell,
+    collections::BTreeMap,
+    time::{Duration, Instant},
+};
 
 mod types;
 pub use types::{CaseOutcome, CounterExampleOutcome, FuzzOutcome};
@@ -98,7 +102,17 @@ impl FuzzedExecutor {
         let max_traces_to_collect = std::cmp::max(1, self.config.gas_report_samples) as usize;
         let show_logs = self.config.show_logs;
 
+        // Start a timer if timeout is set.
+        let start_time = start_timer(self.config.timeout);
+
         let run_result = self.runner.clone().run(&strategy, |calldata| {
+            // Check if the timeout has been reached.
+            if let Some((start_time, timeout)) = start_time {
+                if start_time.elapsed() > timeout {
+                    return Err(TestCaseError::fail(TEST_TIMEOUT));
+                }
+            }
+
             let fuzz_res = self.single_fuzz(address, should_fail, calldata)?;
 
             // If running with progress then increment current run.
@@ -193,17 +207,21 @@ impl FuzzedExecutor {
             }
             Err(TestError::Fail(reason, _)) => {
                 let reason = reason.to_string();
-                result.reason = (!reason.is_empty()).then_some(reason);
-
-                let args = if let Some(data) = calldata.get(4..) {
-                    func.abi_decode_input(data, false).unwrap_or_default()
+                if reason == TEST_TIMEOUT {
+                    // If the reason is a timeout, we consider the fuzz test successful.
+                    result.success = true;
                 } else {
-                    vec![]
-                };
+                    result.reason = (!reason.is_empty()).then_some(reason);
+                    let args = if let Some(data) = calldata.get(4..) {
+                        func.abi_decode_input(data, false).unwrap_or_default()
+                    } else {
+                        vec![]
+                    };
 
-                result.counterexample = Some(CounterExample::Single(
-                    BaseCounterExample::from_fuzz_call(calldata, args, call.traces),
-                ));
+                    result.counterexample = Some(CounterExample::Single(
+                        BaseCounterExample::from_fuzz_call(calldata, args, call.traces),
+                    ));
+                }
             }
         }
 
@@ -270,3 +288,8 @@ impl FuzzedExecutor {
         }
     }
 }
+
+/// Starts timer for fuzz test, if any timeout configured.
+pub(crate) fn start_timer(timeout: Option<u32>) -> Option<(Instant, Duration)> {
+    timeout.map(|timeout| (Instant::now(), Duration::from_secs(timeout.into())))
+}

crates/evm/evm/src/executors/invariant/mod.rs

@@ -10,6 +10,7 @@ use foundry_config::InvariantConfig;
 use foundry_evm_core::{
     constants::{
         CALLER, CHEATCODE_ADDRESS, DEFAULT_CREATE2_DEPLOYER, HARDHAT_CONSOLE_ADDRESS, MAGIC_ASSUME,
+        TEST_TIMEOUT,
     },
     precompiles::PRECOMPILES,
 };
@@ -49,7 +50,7 @@ pub use result::InvariantFuzzTestResult;
 use serde::{Deserialize, Serialize};
 
 mod shrink;
-use crate::executors::EvmError;
+use crate::executors::{fuzz::start_timer, EvmError};
 pub use shrink::check_sequence;
 
 sol! {
@@ -332,6 +333,9 @@ impl<'a> InvariantExecutor<'a> {
         let (invariant_test, invariant_strategy) =
             self.prepare_test(&invariant_contract, fuzz_fixtures)?;
 
+        // Start a timer if timeout is set.
+        let start_time = start_timer(self.config.timeout);
+
         let _ = self.runner.run(&invariant_strategy, |first_input| {
             // Create current invariant run data.
             let mut current_run = InvariantTestRun::new(
@@ -347,6 +351,17 @@ impl<'a> InvariantExecutor<'a> {
             }
 
             while current_run.depth < self.config.depth {
+                // Check if the timeout has been reached.
+                if let Some((start_time, timeout)) = start_time {
+                    if start_time.elapsed() > timeout {
+                        // Since we never record a revert here the test is still considered
+                        // successful even though it timed out. We *want*
+                        // this behavior for now, so that's ok, but
+                        // future developers should be aware of this.
+                        return Err(TestCaseError::fail(TEST_TIMEOUT));
+                    }
+                }
+
                 let tx = current_run.inputs.last().ok_or_else(|| {
                     TestCaseError::fail("No input generated to call fuzzed target.")
                 })?;

crates/forge/bin/cmd/test/mod.rs

@@ -145,6 +145,10 @@ pub struct TestArgs {
     #[arg(long, env = "FOUNDRY_FUZZ_RUNS", value_name = "RUNS")]
     pub fuzz_runs: Option<u64>,
 
+    /// Timeout for each fuzz run in seconds.
+    #[arg(long, env = "FOUNDRY_FUZZ_TIMEOUT", value_name = "TIMEOUT")]
+    pub fuzz_timeout: Option<u64>,
+
     /// File to rerun fuzz failures from.
     #[arg(long)]
     pub fuzz_input_file: Option<String>,
@@ -870,6 +874,9 @@ impl Provider for TestArgs {
         if let Some(fuzz_runs) = self.fuzz_runs {
             fuzz_dict.insert("runs".to_string(), fuzz_runs.into());
         }
+        if let Some(fuzz_timeout) = self.fuzz_timeout {
+            fuzz_dict.insert("timeout".to_string(), fuzz_timeout.into());
+        }
         if let Some(fuzz_input_file) = self.fuzz_input_file.clone() {
             fuzz_dict.insert("failure_persist_file".to_string(), fuzz_input_file.into());
         }

crates/forge/tests/it/fuzz.rs

@@ -235,3 +235,38 @@ contract InlineMaxRejectsTest is Test {
 ...
 "#]]);
 });
+
+// Tests that test timeout config is properly applied.
+// If test doesn't timeout after one second, then test will fail with `rejected too many inputs`.
+forgetest_init!(test_fuzz_timeout, |prj, cmd| {
+    prj.wipe_contracts();
+
+    prj.add_test(
+        "Contract.t.sol",
+        r#"
+import {Test} from "forge-std/Test.sol";
+
+contract FuzzTimeoutTest is Test {
+    /// forge-config: default.fuzz.max-test-rejects = 10000
+    /// forge-config: default.fuzz.timeout = 1
+    function test_fuzz_bound(uint256 a) public pure {
+        vm.assume(a == 0);
+    }
+}
+   "#,
+    )
+    .unwrap();
+
+    cmd.args(["test"]).assert_success().stdout_eq(str![[r#"
+[COMPILING_FILES] with [SOLC_VERSION]
+[SOLC_VERSION] [ELAPSED]
+Compiler run successful!
+
+Ran 1 test for test/Contract.t.sol:FuzzTimeoutTest
+[PASS] test_fuzz_bound(uint256) (runs: [..], [AVG_GAS])
+Suite result: ok. 1 passed; 0 failed; 0 skipped; [ELAPSED]
+
+Ran 1 test suite [ELAPSED]: 1 tests passed, 0 failed, 0 skipped (1 total tests)
+
+"#]]);
+});

crates/forge/tests/it/invariant.rs

@@ -928,3 +928,53 @@ Ran 2 tests for test/SelectorMetricsTest.t.sol:CounterTest
 ...
 "#]]);
 });
+
+// Tests that invariant exists with success after configured timeout.
+forgetest_init!(should_apply_configured_timeout, |prj, cmd| {
+    // Add initial test that breaks invariant.
+    prj.add_test(
+        "TimeoutTest.t.sol",
+        r#"
+import {Test} from "forge-std/Test.sol";
+
+contract TimeoutHandler is Test {
+    uint256 public count;
+
+    function increment() public {
+        count++;
+    }
+}
+
+contract TimeoutTest is Test {
+    TimeoutHandler handler;
+
+    function setUp() public {
+        handler = new TimeoutHandler();
+    }
+
+    /// forge-config: default.invariant.runs = 10000
+    /// forge-config: default.invariant.depth = 20000
+    /// forge-config: default.invariant.timeout = 1
+    function invariant_counter_timeout() public view {
+        // Invariant will fail if more than 10000 increments.
+        // Make sure test timeouts after one second and remaining runs are canceled.
+        require(handler.count() < 10000);
+    }
+}
+     "#,
+    )
+    .unwrap();
+
+    cmd.args(["test", "--mt", "invariant_counter_timeout"]).assert_success().stdout_eq(str![[r#"
+[COMPILING_FILES] with [SOLC_VERSION]
+[SOLC_VERSION] [ELAPSED]
+Compiler run successful!
+
+Ran 1 test for test/TimeoutTest.t.sol:TimeoutTest
+[PASS] invariant_counter_timeout() (runs: 0, calls: 0, reverts: 0)
+Suite result: ok. 1 passed; 0 failed; 0 skipped; [ELAPSED]
+
+Ran 1 test suite [ELAPSED]: 1 tests passed, 0 failed, 0 skipped (1 total tests)
+
+"#]]);
+});

crates/forge/tests/it/test_helpers.rs

@@ -95,6 +95,7 @@ impl ForgeTestProfile {
                 failure_persist_dir: Some(tempfile::tempdir().unwrap().into_path()),
                 failure_persist_file: Some("testfailure".to_string()),
                 show_logs: false,
+                timeout: None,
             })
             .invariant(InvariantConfig {
                 runs: 256,
@@ -113,6 +114,7 @@ impl ForgeTestProfile {
                 gas_report_samples: 256,
                 failure_persist_dir: Some(tempfile::tempdir().unwrap().into_path()),
                 show_metrics: false,
+                timeout: None,
             })
             .build(output, Path::new(self.project().root()))
             .expect("Config loaded")