Merge pull request #1952 from madhav-madhusoodanan/intrinsic-test-updates

`intrinsic-test`: Coalescing all intrinsic tests into a single run of the test-file binaries

Author: folkertdev

ci/intrinsic-test.sh

@@ -75,7 +75,7 @@ case ${TARGET} in
         TEST_CXX_COMPILER="clang++"
         TEST_RUNNER="${CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER}"
         TEST_SKIP_INTRINSICS=crates/intrinsic-test/missing_x86.txt
-        : "${TEST_SAMPLE_INTRINSICS_PERCENTAGE:=5}"
+        : "${TEST_SAMPLE_INTRINSICS_PERCENTAGE:=20}"
         ;;
     *)
         ;;

crates/intrinsic-test/src/common/cli.rs

@@ -7,12 +7,6 @@ pub enum Language {
     C,
 }
 
-pub enum FailureReason {
-    RunC(String),
-    RunRust(String),
-    Difference(String, String, String),
-}
-
 /// Intrinsic test tool
 #[derive(clap::Parser)]
 #[command(

crates/intrinsic-test/src/common/compare.rs

@@ -1,7 +1,8 @@
-use super::cli::FailureReason;
+use itertools::Itertools;
 use rayon::prelude::*;
-use std::process::Command;
+use std::{collections::HashMap, process::Command};
 
+pub const INTRINSIC_DELIMITER: &str = "############";
 fn runner_command(runner: &str) -> Command {
     let mut it = runner.split_whitespace();
     let mut cmd = Command::new(it.next().unwrap());
@@ -11,85 +12,106 @@ fn runner_command(runner: &str) -> Command {
 }
 
 pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target: &str) -> bool {
-    let intrinsics = intrinsic_name_list
-        .par_iter()
-        .filter_map(|intrinsic_name| {
-            let c = runner_command(runner)
+    let (c, rust) = rayon::join(
+        || {
+            runner_command(runner)
                 .arg("./intrinsic-test-programs")
-                .arg(intrinsic_name)
                 .current_dir("c_programs")
-                .output();
-
-            let rust = runner_command(runner)
+                .output()
+        },
+        || {
+            runner_command(runner)
                 .arg(format!("./target/{target}/release/intrinsic-test-programs"))
-                .arg(intrinsic_name)
                 .current_dir("rust_programs")
-                .output();
+                .output()
+        },
+    );
+    let (c, rust) = match (c, rust) {
+        (Ok(c), Ok(rust)) => (c, rust),
+        failure => panic!("Failed to run: {failure:#?}"),
+    };
 
-            let (c, rust) = match (c, rust) {
-                (Ok(c), Ok(rust)) => (c, rust),
-                a => panic!("{a:#?}"),
-            };
+    if !c.status.success() {
+        error!(
+            "Failed to run C program.\nstdout: {stdout}\nstderr: {stderr}",
+            stdout = std::str::from_utf8(&c.stdout).unwrap_or(""),
+            stderr = std::str::from_utf8(&c.stderr).unwrap_or(""),
+        );
+    }
 
-            if !c.status.success() {
-                error!(
-                    "Failed to run C program for intrinsic `{intrinsic_name}`\nstdout: {stdout}\nstderr: {stderr}",
-                    stdout = std::str::from_utf8(&c.stdout).unwrap_or(""),
-                    stderr = std::str::from_utf8(&c.stderr).unwrap_or(""),
-                );
-                return Some(FailureReason::RunC(intrinsic_name.clone()));
-            }
+    if !rust.status.success() {
+        error!(
+            "Failed to run Rust program.\nstdout: {stdout}\nstderr: {stderr}",
+            stdout = std::str::from_utf8(&rust.stdout).unwrap_or(""),
+            stderr = std::str::from_utf8(&rust.stderr).unwrap_or(""),
+        );
+    }
 
-            if !rust.status.success() {
-                error!(
-                    "Failed to run Rust program for intrinsic `{intrinsic_name}`\nstdout: {stdout}\nstderr: {stderr}",
-                    stdout = std::str::from_utf8(&rust.stdout).unwrap_or(""),
-                    stderr = std::str::from_utf8(&rust.stderr).unwrap_or(""),
-                );
-                return Some(FailureReason::RunRust(intrinsic_name.clone()));
-            }
+    info!("Completed running C++ and Rust test binaries");
+    let c = std::str::from_utf8(&c.stdout)
+        .unwrap()
+        .to_lowercase()
+        .replace("-nan", "nan");
+    let rust = std::str::from_utf8(&rust.stdout)
+        .unwrap()
+        .to_lowercase()
+        .replace("-nan", "nan");
 
-            info!("Comparing intrinsic: {intrinsic_name}");
+    let c_output_map = c
+        .split(INTRINSIC_DELIMITER)
+        .filter_map(|output| output.trim().split_once("\n"))
+        .collect::<HashMap<&str, &str>>();
+    let rust_output_map = rust
+        .split(INTRINSIC_DELIMITER)
+        .filter_map(|output| output.trim().split_once("\n"))
+        .collect::<HashMap<&str, &str>>();
 
-            let c = std::str::from_utf8(&c.stdout)
-                .unwrap()
-                .to_lowercase()
-                .replace("-nan", "nan");
-            let rust = std::str::from_utf8(&rust.stdout)
-                .unwrap()
-                .to_lowercase()
-                .replace("-nan", "nan");
+    let intrinsics = c_output_map
+        .keys()
+        .chain(rust_output_map.keys())
+        .unique()
+        .collect_vec();
 
-            if c == rust {
+    info!("Comparing outputs");
+    let intrinsics_diff_count = intrinsics
+        .par_iter()
+        .filter_map(|&&intrinsic| {
+            let c_output = c_output_map.get(intrinsic).unwrap();
+            let rust_output = rust_output_map.get(intrinsic).unwrap();
+            if rust_output.eq(c_output) {
                 None
             } else {
-                Some(FailureReason::Difference(intrinsic_name.clone(), c, rust))
+                let diff = diff::lines(c_output, rust_output);
+                let diffs = diff
+                    .into_iter()
+                    .filter_map(|diff| match diff {
+                        diff::Result::Left(_) | diff::Result::Right(_) => Some(diff),
+                        diff::Result::Both(_, _) => None,
+                    })
+                    .collect_vec();
+                if diffs.len() > 0 {
+                    Some((intrinsic, diffs))
+                } else {
+                    None
+                }
             }
         })
-        .collect::<Vec<_>>();
-
-    intrinsics.iter().for_each(|reason| match reason {
-        FailureReason::Difference(intrinsic, c, rust) => {
+        .inspect(|(intrinsic, diffs)| {
             println!("Difference for intrinsic: {intrinsic}");
-            let diff = diff::lines(c, rust);
-            diff.iter().for_each(|diff| match diff {
+            diffs.into_iter().for_each(|diff| match diff {
                 diff::Result::Left(c) => println!("C: {c}"),
                 diff::Result::Right(rust) => println!("Rust: {rust}"),
-                diff::Result::Both(_, _) => (),
+                _ => (),
             });
             println!("****************************************************************");
-        }
-        FailureReason::RunC(intrinsic) => {
-            println!("Failed to run C program for intrinsic {intrinsic}")
-        }
-        FailureReason::RunRust(intrinsic) => {
-            println!("Failed to run rust program for intrinsic {intrinsic}")
-        }
-    });
+        })
+        .count();
+
     println!(
         "{} differences found (tested {} intrinsics)",
-        intrinsics.len(),
+        intrinsics_diff_count,
         intrinsic_name_list.len()
     );
-    intrinsics.is_empty()
+
+    intrinsics_diff_count == 0
 }

crates/intrinsic-test/src/common/gen_c.rs

@@ -1,6 +1,7 @@
 use crate::common::intrinsic::Intrinsic;
 
 use super::argument::Argument;
+use super::compare::INTRINSIC_DELIMITER;
 use super::indentation::Indentation;
 use super::intrinsic_helpers::IntrinsicTypeDefinition;
 
@@ -147,22 +148,17 @@ pub fn write_main_cpp<'a>(
     }
 
     writeln!(w, "int main(int argc, char **argv) {{")?;
-    writeln!(w, "    std::string intrinsic_name = argv[1];")?;
-
-    writeln!(w, "    if (false) {{")?;
 
     for intrinsic in intrinsics {
-        writeln!(w, "    }} else if (intrinsic_name == \"{intrinsic}\") {{")?;
-        writeln!(w, "        return run_{intrinsic}();")?;
+        writeln!(
+            w,
+            "    std::cout << \"{INTRINSIC_DELIMITER}\" << std::endl;"
+        )?;
+        writeln!(w, "    std::cout << \"{intrinsic}\" << std::endl;")?;
+        writeln!(w, "    run_{intrinsic}();\n")?;
     }
 
-    writeln!(w, "    }} else {{")?;
-    writeln!(
-        w,
-        "        std::cerr << \"Unknown command: \" << intrinsic_name << \"\\n\";"
-    )?;
-    writeln!(w, "        return -1;")?;
-    writeln!(w, "    }}")?;
+    writeln!(w, "    return 0;")?;
 
     writeln!(w, "}}")?;
 

crates/intrinsic-test/src/common/gen_rust.rs

@@ -1,11 +1,11 @@
 use itertools::Itertools;
 use std::process::Command;
 
-use crate::common::argument::ArgumentList;
-use crate::common::intrinsic::Intrinsic;
-
+use super::compare::INTRINSIC_DELIMITER;
 use super::indentation::Indentation;
 use super::intrinsic_helpers::IntrinsicTypeDefinition;
+use crate::common::argument::ArgumentList;
+use crate::common::intrinsic::Intrinsic;
 
 // The number of times each intrinsic will be called.
 pub(crate) const PASSES: u32 = 20;
@@ -86,18 +86,12 @@ pub fn write_main_rs<'a>(
 
     writeln!(w, "fn main() {{")?;
 
-    writeln!(w, "    match std::env::args().nth(1).unwrap().as_str() {{")?;
-
     for binary in intrinsics {
-        writeln!(w, "        \"{binary}\" => run_{binary}(),")?;
+        writeln!(w, "    println!(\"{INTRINSIC_DELIMITER}\");")?;
+        writeln!(w, "    println!(\"{binary}\");")?;
+        writeln!(w, "    run_{binary}();\n")?;
     }
 
-    writeln!(
-        w,
-        "        other => panic!(\"unknown intrinsic `{{}}`\", other),"
-    )?;
-
-    writeln!(w, "    }}")?;
     writeln!(w, "}}")?;
 
     Ok(())

crates/intrinsic-test/src/main.rs

@@ -34,7 +34,7 @@ fn run(test_environment: impl SupportedArchitectureTest) {
     if !test_environment.build_rust_file() {
         std::process::exit(3);
     }
-    info!("comparing outputs");
+    info!("Running binaries");
     if !test_environment.compare_outputs() {
         std::process::exit(1);
     }