feat: bless test

Author: Wybxc

bootstrap/Cargo.lock

@@ -10,4 +10,6 @@ color-print = "0.3.6"
 env_logger = "0.11.5"
 glob = "0.3.1"
 log = "0.4.22"
+regex = "1.11.1"
+similar = "2.6.0"
 which = "6.0.1"

bootstrap/Cargo.toml

@@ -5,14 +5,19 @@ use anstream::{eprint as print, eprintln as println};
 use clap::Args;
 use color_print::{cprint, cprintln};
 use glob::glob;
+use similar::{ChangeTag, TextDiff};
 use which::which;
 
 use crate::manifest::Manifest;
 use crate::Run;
 
 /// Run tests
 #[derive(Args, Debug)]
-pub struct TestCommand {}
+pub struct TestCommand {
+    /// Update the blessed output
+    #[clap(long)]
+    pub bless: bool,
+}
 
 impl Run for TestCommand {
     fn run(&self, manifest: &Manifest) {
@@ -37,7 +42,12 @@ impl Run for TestCommand {
                 TestType::FileCheck => {
                     cprint!("File checking {}...", testcase.name);
                     testcase.build(manifest);
-                    filechecker.run(&testcase.source, &testcase.output_file);
+                    filechecker.run(&testcase);
+                }
+                TestType::Bless => {
+                    cprint!("Blessing {}...", testcase.name);
+                    testcase.build(manifest);
+                    bless(self.bless, &testcase);
                 }
                 TestType::Compile => {
                     cprint!("Compiling {}...", testcase.name);
@@ -55,24 +65,15 @@ impl Run for TestCommand {
 
 impl TestCommand {
     pub fn collect_testcases(&self, manifest: &Manifest) -> Vec<TestCase> {
-        let mut result = vec![];
-
-        // Test auxiliary (should compile first)
-        for case in glob("tests/auxiliary/*.rs").unwrap() {
-            let case = case.unwrap();
-            let filename = case.file_stem().unwrap();
-            let name = format!("auxiliary/{}", filename.to_string_lossy());
-            let output_file = manifest.out_dir.join(filename);
-            result.push(TestCase { name, source: case, output_file, test: TestType::CompileLib })
-        }
+        let mut tests = vec![];
 
         // Examples
         for case in glob("examples/*.rs").unwrap() {
             let case = case.unwrap();
             let filename = case.file_stem().unwrap();
             let name = format!("examples/{}", filename.to_string_lossy());
             let output_file = manifest.out_dir.join("examples").join(filename);
-            result.push(TestCase { name, source: case, output_file, test: TestType::Compile })
+            tests.push(TestCase { name, source: case, output_file, test: TestType::Compile })
         }
 
         // Codegen tests
@@ -81,18 +82,49 @@ impl TestCommand {
             let filename = case.file_stem().unwrap();
             let name = format!("codegen/{}", filename.to_string_lossy());
             let output_file = manifest.out_dir.join("tests/codegen").join(filename);
-            result.push(TestCase { name, source: case, output_file, test: TestType::FileCheck })
+            tests.push(TestCase { name, source: case, output_file, test: TestType::FileCheck })
+        }
+
+        // Bless tests - the output should be the same as the last run
+        for case in glob("tests/bless/*.rs").unwrap() {
+            let case = case.unwrap();
+            let filename = case.file_stem().unwrap();
+            let name = format!("bless/{}", filename.to_string_lossy());
+            let output_file = manifest.out_dir.join("tests/bless").join(filename);
+            tests.push(TestCase { name, source: case, output_file, test: TestType::Bless })
         }
 
-        result
+        // Collect test-auxiliary
+        let aux_use = regex::Regex::new(r"^//@\s*aux-build:(?P<fname>.*)").unwrap();
+        let mut auxiliary = vec![];
+        for case in tests.iter() {
+            let source = std::fs::read_to_string(&case.source).unwrap();
+            for cap in aux_use.captures_iter(&source) {
+                let fname = cap.name("fname").unwrap().as_str();
+                let source = Path::new("tests/auxiliary").join(fname);
+                let filename = source.file_stem().unwrap();
+                let name = format!("auxiliary/{}", filename.to_string_lossy());
+                let output_file = manifest.out_dir.join(filename); // aux files are output to the base directory
+                auxiliary.push(TestCase { name, source, output_file, test: TestType::CompileLib })
+            }
+        }
+
+        // Compile auxiliary before the tests
+        let mut cases = auxiliary;
+        cases.extend(tests);
+        cases
     }
 }
 
 pub enum TestType {
     /// Test an executable can be compiled
     Compile,
+    /// Test a library can be compiled
     CompileLib,
+    /// Run LLVM FileCheck on the generated code
     FileCheck,
+    /// Bless test - the output should be the same as the last run
+    Bless,
 }
 
 pub struct TestCase {
@@ -125,11 +157,27 @@ impl TestCase {
             .args(["--crate-type", "lib"])
             .arg("-O")
             .arg(&self.source)
-            .arg("--out-dir")
-            .arg(self.output_file.parent().unwrap());
+            .arg("--out-dir") // we use `--out-dir` to integrate with the default name convention
+            .arg(output_dir); // so here we ignore the filename and just use the directory
         log::debug!("running {:?}", command);
         command.status().unwrap();
     }
+
+    /// Get the generated C file f
+    pub fn generated(&self) -> PathBuf {
+        let case = self.source.file_stem().unwrap().to_string_lossy();
+        let generated = std::fs::read_dir(self.output_file.parent().unwrap())
+            .unwrap()
+            .filter_map(|entry| entry.ok())
+            .find(|entry| {
+                let filename = entry.file_name();
+                let filename = filename.to_string_lossy();
+                filename.ends_with(".c") && filename.starts_with(case.as_ref())
+            });
+
+        assert!(generated.is_some(), "could not find {case}'s generated file");
+        generated.unwrap().path()
+    }
 }
 
 struct FileChecker {
@@ -153,25 +201,41 @@ impl FileChecker {
         Self { filecheck }
     }
 
-    fn run(&self, source: &Path, output: &Path) {
-        let case = source.file_stem().unwrap().to_string_lossy();
-        let generated = std::fs::read_dir(output.parent().unwrap())
-            .unwrap()
-            .filter_map(|entry| entry.ok())
-            .find(|entry| {
-                let filename = entry.file_name();
-                let filename = filename.to_string_lossy();
-                filename.ends_with(".c") && filename.starts_with(case.as_ref())
-            });
-
-        assert!(generated.is_some(), "could not find {case}'s generated file");
-        let generated = generated.unwrap();
-
-        let generated = File::open(generated.path()).unwrap();
+    fn run(&self, case: &TestCase) {
+        let generated = File::open(case.generated()).unwrap();
         let mut command = std::process::Command::new(&self.filecheck);
-        command.arg(source).stdin(generated);
+        command.arg(&case.source).stdin(generated);
         log::debug!("running {:?}", command);
         let output = command.output().unwrap();
-        assert!(output.status.success(), "failed to run FileCheck on {case}");
+        assert!(
+            output.status.success(),
+            "failed to run FileCheck on {}",
+            case.source.file_stem().unwrap().to_string_lossy()
+        );
+    }
+}
+
+fn bless(update: bool, case: &TestCase) {
+    let output = case.generated();
+    let blessed = case.source.with_extension("c");
+    if update {
+        std::fs::copy(output, blessed).unwrap();
+    } else {
+        let output = std::fs::read_to_string(output).unwrap();
+        let blessed = std::fs::read_to_string(blessed).unwrap();
+
+        let diff = TextDiff::from_lines(&blessed, &output);
+        if diff.ratio() < 1.0 {
+            cprintln!("<r,s>output does not match blessed output</r,s>");
+            for change in diff.iter_all_changes() {
+                let lineno = change.old_index().unwrap_or(change.new_index().unwrap_or(0));
+                match change.tag() {
+                    ChangeTag::Equal => print!(" {:4}| {}", lineno, change),
+                    ChangeTag::Insert => cprint!("<g>+{:4}| {}</g>", lineno, change),
+                    ChangeTag::Delete => cprint!("<r>-{:4}| {}</r>", lineno, change),
+                }
+            }
+            std::process::exit(1);
+        }
     }
 }

bootstrap/src/test.rs

@@ -1,3 +1,5 @@
+//@ aux-build:mini_core.rs
+
 #![feature(no_core)]
 #![no_core]
 #![no_main]

examples/basic_math.rs

@@ -0,0 +1,17 @@
+//@ aux-build:mini_core.rs
+
+#![feature(no_core)]
+#![no_core]
+#![no_main]
+
+extern crate mini_core;
+
+#[no_mangle]
+pub fn main() -> i32 {
+    0
+}
+
+#[no_mangle]
+pub fn foo(x: u8, _y: u16, _z: u32) -> i64 {
+    x as i64
+}

tests/bless/basic_math.rs

@@ -1,5 +1,7 @@
 //! Test that the generated code has the filename and function name in it
 
+//@ aux-build:mini_core.rs
+
 // CHECK: filename
 
 #![feature(no_core)]

tests/codegen/filename.rs

@@ -1,5 +1,7 @@
 //! Test that the generated code has the right number of parameters
 
+//@ aux-build:mini_core.rs
+
 #![feature(no_core)]
 #![no_core]
 #![no_main]
@@ -11,7 +13,7 @@ extern crate mini_core;
 
 // expect three int params
 // CHECK-LABEL: foo
-// CHECK: (int32_t {{[a-zA-Z_][a-zA-Z0-9_]*}}, int32_t {{[a-zA-Z_][a-zA-Z0-9_]*}}, int32_t {{[a-zA-Z_][a-zA-Z0-9_]*}})
+// CHECK: (int32_t {{[[:alnum:]_]*}}, int32_t {{[[:alnum:]_]*}}, int32_t {{[[:alnum:]_]*}})
 // CHECK: return 0;
 #[no_mangle]
 pub fn foo(_x: i32, _y: i32, _z: i32) -> i32 {

tests/codegen/params_count.rs

@@ -1,12 +1,15 @@
 //! Test that we can return a value from a function
+
+//@ aux-build:mini_core.rs
+
 #![feature(no_core)]
 #![no_core]
 #![no_main]
 
 extern crate mini_core;
 
 // CHECK-LABEL: main
-// CHECK: 42
+// CHECK: return 42;
 #[no_mangle]
 pub fn main() -> i32 {
     42