From 1a6508695e787ab82ac63426d715705996bb48cd Mon Sep 17 00:00:00 2001 From: Firestar99 Date: Mon, 14 Jul 2025 19:06:49 +0200 Subject: [PATCH 01/17] difftest: use relative paths, create output directory in target --- tests/difftests/bin/Cargo.toml | 4 +- tests/difftests/bin/src/main.rs | 56 ++++--- tests/difftests/bin/src/runner.rs | 229 +++++++--------------------- tests/difftests/bin/src/testcase.rs | 175 +++++++++++++++++++++ 4 files changed, 265 insertions(+), 199 deletions(-) create mode 100644 tests/difftests/bin/src/testcase.rs diff --git a/tests/difftests/bin/Cargo.toml b/tests/difftests/bin/Cargo.toml index a8780c8938..6eb5ca6613 100644 --- a/tests/difftests/bin/Cargo.toml +++ b/tests/difftests/bin/Cargo.toml @@ -16,7 +16,6 @@ use-compiled-tools = [] anyhow = "1.0" tracing = "0.1" tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"] } -tempfile = "3.5" tester = "0.9.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -27,5 +26,8 @@ bytemuck = "1.21.0" difftest = { path = "../lib" } tabled = { version = "0.20.0", default-features = false, features = ["std"] } +[dev-dependencies] +tempfile = "3.5" + [lints] workspace = true diff --git a/tests/difftests/bin/src/main.rs b/tests/difftests/bin/src/main.rs index 58614d5a8b..6856f238a4 100644 --- a/tests/difftests/bin/src/main.rs +++ b/tests/difftests/bin/src/main.rs @@ -1,8 +1,10 @@ #![allow(clippy::exit)] +use crate::testcase::collect_test_dirs; use anyhow::Result; +use runner::Runner; use std::{ - env, + env, fs, process::{self, Command}, }; use tester::{ @@ -14,7 +16,7 @@ use tracing_subscriber::FmtSubscriber; mod differ; mod runner; -use runner::Runner; +mod testcase; fn main() -> Result<()> { let subscriber = FmtSubscriber::builder() @@ -63,10 +65,19 @@ fn main() -> Result<()> { .expect("Failed to canonicalize tests directory"); tracing::debug!("Using tests directory: {}", base.display()); - let runner = Runner::new(base.clone()); + let output_dir = std::path::Path::new(manifest_dir).join("../tests/target/difftest"); + fs::create_dir_all(&output_dir)?; + let output_dir = output_dir + .canonicalize() + .expect("Failed to canonicalize tests directory"); + tracing::debug!("Using output directory: {}", output_dir.display()); + + let runner = Runner { + base_dir: base.clone(), + output_dir, + }; - let test_cases = - Runner::collect_test_dirs(&base).expect("Failed to collect test case directories"); + let test_cases = collect_test_dirs(&base).expect("Failed to collect test case directories"); if test_cases.is_empty() { eprintln!("No valid tests found in {}", base.display()); process::exit(1); @@ -92,25 +103,22 @@ fn main() -> Result<()> { let tests: Vec = test_cases .into_iter() - .map(|case| { - let test_name = Runner::format_test_name(&case, &base); - TestDescAndFn { - desc: TestDesc { - name: DynTestName(test_name.clone()), - ignore: false, - should_panic: ShouldPanic::No, - allow_fail: false, - test_type: TestType::IntegrationTest, - }, - testfn: TestFn::DynTestFn(Box::new({ - let runner = runner.clone(); - move || { - runner - .run_test_case(&case) - .unwrap_or_else(|e| panic!("{}", e)); - } - })), - } + .map(|case| TestDescAndFn { + desc: TestDesc { + name: DynTestName(case.to_string()), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + test_type: TestType::IntegrationTest, + }, + testfn: TestFn::DynTestFn(Box::new({ + let runner = runner.clone(); + move || { + runner + .run_test_case(&case) + .unwrap_or_else(|e| panic!("{}", e)); + } + })), }) .collect(); diff --git a/tests/difftests/bin/src/runner.rs b/tests/difftests/bin/src/runner.rs index 13738edcd2..f0e4052921 100644 --- a/tests/difftests/bin/src/runner.rs +++ b/tests/difftests/bin/src/runner.rs @@ -4,15 +4,14 @@ use serde::{Deserialize, Serialize}; use std::{ collections::{HashMap, HashSet}, fs, - io::Write, - path::{Component, Path, PathBuf}, + path::{Path, PathBuf}, process::Command, }; -use tempfile::NamedTempFile; use thiserror::Error; use tracing::{debug, error, info, trace}; use crate::differ::{DiffMagnitude, Difference, DifferenceDisplay, OutputDiffer}; +use crate::testcase::TestCase; #[derive(Debug, Error)] pub enum RunnerError { @@ -214,65 +213,54 @@ impl ErrorReport { #[derive(Clone)] pub struct Runner { pub base_dir: PathBuf, + pub output_dir: PathBuf, } impl Runner { - pub fn new(base_dir: PathBuf) -> Self { - Self { base_dir } - } - - pub fn run_test_case(&self, test_case: &Path) -> RunnerResult<()> { - trace!("Starting test case: {}", test_case.display()); - let packages = self.collect_packages(test_case)?; + pub fn run_test_case(&self, test_case: &TestCase) -> RunnerResult<()> { + trace!("Starting test case: {}", test_case); debug!( "Found {} package(s) in test case {}", - packages.len(), - test_case.display() + test_case.test_binaries.len(), + test_case ); - if packages.len() < 2 { - error!("Insufficient packages in test case {}", test_case.display()); + if test_case.test_binaries.len() < 2 { + error!("Insufficient packages in test case {}", test_case); return Err(RunnerError::InsufficientPackages { - count: packages.len(), + count: test_case.test_binaries.len(), }); } // Pre-check that package names are globally unique. let mut names_seen = HashSet::new(); - for package in &packages { - let manifest_path = package.join("Cargo.toml"); + for package in &test_case.test_binaries { + let manifest_path = package.absolute_path.join("Cargo.toml"); let pkg_name = self.get_package_name(&manifest_path)?; if !names_seen.insert(pkg_name.clone()) { return Err(RunnerError::DuplicatePackageName { pkg_name }); } } - let mut temp_files: Vec = Vec::with_capacity(packages.len()); - let mut pkg_outputs: Vec = Vec::with_capacity(packages.len()); + let mut pkg_outputs: Vec = Vec::with_capacity(test_case.test_binaries.len()); let mut epsilon: Option = None; let mut output_type = None; - for package in packages { - trace!("Processing package at {}", package.display()); - let manifest_path = package.join("Cargo.toml"); + for package in &test_case.test_binaries { + trace!( + "Processing package '{}' at '{}'", + package, + package.absolute_path.display() + ); + let manifest_path = package.absolute_path.join("Cargo.toml"); let pkg_name = self.get_package_name(&manifest_path)?; debug!("Package '{}' detected", pkg_name); - let output_file = NamedTempFile::new()?; - let temp_output_path = output_file.path().to_path_buf(); - temp_files.push(output_file); - - let metadata_file = NamedTempFile::new()?; - let temp_metadata_path = metadata_file.path().to_path_buf(); - temp_files.push(metadata_file); - - trace!( - "Temporary output file created at {}", - temp_output_path.display() - ); - trace!( - "Temporary metadata file created at {}", - temp_metadata_path.display() - ); + let package_out = self.output_dir.join(package.path_from_root()); + fs::create_dir_all(&package_out)?; + debug!("Writing output to '{}'", package_out.display()); + let config_file = package_out.join("config.json"); + let temp_output_path = package_out.join("out.bin"); + let temp_metadata_path = package_out.join("metadata.json"); let config = HarnessConfig { output_path: temp_output_path.clone(), @@ -280,9 +268,8 @@ impl Runner { }; let config_json = serde_json::to_string(&config) .map_err(|e| RunnerError::Config { msg: e.to_string() })?; - let mut config_file = NamedTempFile::new()?; - write!(config_file, "{config_json}").map_err(|e| RunnerError::Io { source: e })?; - trace!("Config file created at {}", config_file.path().display()); + fs::write(&config_file, &config_json)?; + trace!("Config file created at {}", config_file.display()); let mut cmd = Command::new("cargo"); cmd.arg("run").arg("--release").arg("--manifest-path").arg( @@ -293,18 +280,14 @@ impl Runner { })?, ); forward_features(&mut cmd); - cmd.arg("--").arg( - config_file - .path() - .to_str() - .ok_or_else(|| RunnerError::Config { - msg: "Invalid config file path".into(), - })?, - ); + cmd.arg("--") + .arg(config_file.to_str().ok_or_else(|| RunnerError::Config { + msg: "Invalid config file path".into(), + })?); debug!("Running cargo command: {:?}", cmd); let output = cmd - .current_dir(&package) + .current_dir(&package.absolute_path) .output() .map_err(|e| RunnerError::Io { source: e })?; let exit_code = output.status.code().unwrap_or(-1); @@ -317,7 +300,7 @@ impl Runner { error!("Cargo execution failed for package '{}'", pkg_name); return Err(RunnerError::CargoExecutionFailed { pkg_name, - package_path: package, + package_path: package.absolute_path.clone(), exit_status: exit_code, stderr: stderr_str, }); @@ -388,7 +371,7 @@ impl Runner { pkg_outputs.push(PackageOutput { pkg_name, - package_path: package, + package_path: package.absolute_path.clone(), output: output_bytes, temp_path: temp_output_path, }); @@ -424,30 +407,12 @@ impl Runner { // Generate detailed error report let details = self.format_error(&pkg_outputs, epsilon, output_type, &*differ, &*display); - self.keep_temp_files(&mut temp_files); return Err(RunnerError::DifferingOutput(details)); } - info!( - "Test case '{}' passed.", - Runner::format_test_name(test_case, test_case.parent().unwrap_or(test_case)) - ); + info!("Test case '{test_case}' passed."); Ok(()) } - #[allow(clippy::unused_self)] - fn collect_packages(&self, test_case: &Path) -> RunnerResult> { - let mut packages = Vec::new(); - for entry in fs::read_dir(test_case)? { - let entry = entry?; - let path = entry.path(); - if path.is_dir() && path.join("Cargo.toml").exists() { - debug!("Found package candidate: {}", path.display()); - packages.push(path); - } - } - Ok(packages) - } - #[allow(clippy::unused_self)] fn group_outputs<'a>( &self, @@ -665,60 +630,6 @@ impl Runner { debug!("Package name '{}' found in manifest", manifest.package.name); Ok(manifest.package.name) } - - #[allow(clippy::unused_self)] - fn keep_temp_files(&self, temp_files: &mut Vec) { - for file in temp_files.drain(..) { - let _ = file.into_temp_path().keep(); - } - } - - pub fn format_test_name(test_case: &Path, base: &Path) -> String { - let name = test_case.strip_prefix(base).map_or_else( - |_| test_case.to_string_lossy().into_owned(), - |relative| { - relative - .components() - .filter_map(|comp| match comp { - Component::Normal(os_str) => Some(os_str.to_string_lossy()), - _ => None, - }) - .collect::>() - .join("::") - }, - ); - format!("difftests::{name}") - } - - pub fn collect_test_dirs(root: &Path) -> RunnerResult> { - let mut test_cases = Vec::new(); - for entry in fs::read_dir(root)? { - let entry = entry?; - let path = entry.path(); - if path.is_dir() { - if Runner::collect_test_binaries(&path)?.len() >= 2 { - debug!("Test case found: {}", path.display()); - test_cases.push(path.clone()); - } - let mut subdirs = Runner::collect_test_dirs(&path)?; - test_cases.append(&mut subdirs); - } - } - Ok(test_cases) - } - - fn collect_test_binaries(test_case: &Path) -> RunnerResult> { - let mut packages = Vec::new(); - for entry in fs::read_dir(test_case)? { - let entry = entry?; - let path = entry.path(); - if path.is_dir() && path.join("Cargo.toml").exists() { - debug!("Found binary package candidate: {}", path.display()); - packages.push(path); - } - } - Ok(packages) - } } pub fn forward_features(cmd: &mut Command) { @@ -740,6 +651,13 @@ mod tests { use std::{fs, io::Write, path::Path, path::PathBuf}; use tempfile::{NamedTempFile, tempdir}; + fn dummy_runner() -> Runner { + Runner { + base_dir: PathBuf::from("dummy_base"), + output_dir: PathBuf::from("dummy_out"), + } + } + fn dummy_package_output(name: &str, path: &str, output: &[u8], temp: &str) -> PackageOutput { PackageOutput { pkg_name: name.to_string(), @@ -755,19 +673,11 @@ mod tests { let pkg2 = dummy_package_output("bar", "/path/to/bar", b"world", "tmp2"); let pkg3 = dummy_package_output("baz", "/path/to/baz", b"hello", "tmp3"); let outputs = vec![pkg1, pkg2, pkg3]; - let runner = Runner::new(PathBuf::from("dummy_base")); + let runner = dummy_runner(); let groups = runner.group_outputs(&outputs, None, OutputType::Raw); assert_eq!(groups.len(), 2); } - #[test] - fn test_format_test_name() { - let base = Path::new("/home/user/tests"); - let test_case = base.join("group1/testcase1"); - let formatted = Runner::format_test_name(&test_case, base); - assert_eq!(formatted, "difftests::group1::testcase1"); - } - #[test] fn test_get_package_name() { let mut temp = NamedTempFile::new().expect("failed to create temp file"); @@ -778,47 +688,13 @@ mod tests { edition = "2021" "#; write!(temp, "{cargo_toml}").expect("failed to write to temp file"); - let runner = Runner::new(PathBuf::from("dummy_base")); + let runner = dummy_runner(); let pkg_name = runner .get_package_name(temp.path()) .expect("failed to get package name"); assert_eq!(pkg_name, "dummy_package"); } - #[test] - fn test_collect_packages() { - let temp_dir = tempdir().expect("failed to create temp dir"); - let dir_path = temp_dir.path(); - let pkg_dir = dir_path.join("pkg1"); - fs::create_dir(&pkg_dir).expect("failed to create pkg1 dir"); - fs::write(pkg_dir.join("Cargo.toml"), "[package]\nname = \"pkg1\"") - .expect("failed to write Cargo.toml"); - let runner = Runner::new(PathBuf::from("dummy_base")); - let packages = runner - .collect_packages(dir_path) - .expect("failed to collect packages"); - assert_eq!(packages.len(), 1); - assert_eq!(packages[0], pkg_dir); - } - - #[test] - fn test_collect_test_dirs() { - let temp_dir = tempdir().expect("failed to create temp dir"); - let base = temp_dir.path(); - let test_case_dir = base.join("test_case"); - fs::create_dir(&test_case_dir).expect("failed to create test_case dir"); - let pkg1_dir = test_case_dir.join("pkg1"); - fs::create_dir(&pkg1_dir).expect("failed to create pkg1"); - fs::write(pkg1_dir.join("Cargo.toml"), "[package]\nname = \"pkg1\"") - .expect("failed to write Cargo.toml for pkg1"); - let pkg2_dir = test_case_dir.join("pkg2"); - fs::create_dir(&pkg2_dir).expect("failed to create pkg2"); - fs::write(pkg2_dir.join("Cargo.toml"), "[package]\nname = \"pkg2\"") - .expect("failed to write Cargo.toml for pkg2"); - let test_dirs = Runner::collect_test_dirs(base).expect("failed to collect test dirs"); - assert!(test_dirs.contains(&test_case_dir)); - } - #[test] fn test_run_test_case_insufficient_packages() { let temp_dir = tempdir().expect("failed to create temp dir"); @@ -828,8 +704,10 @@ mod tests { fs::create_dir(&pkg_dir).expect("failed to create pkg1"); fs::write(pkg_dir.join("Cargo.toml"), "[package]\nname = \"pkg1\"") .expect("failed to write Cargo.toml for pkg1"); - let runner = Runner::new(PathBuf::from("dummy_base")); - let result = runner.run_test_case(&test_case_dir); + let test_case = TestCase::try_new(temp_dir.path(), Path::new("single_pkg")) + .unwrap() + .unwrap(); + let result = dummy_runner().run_test_case(&test_case); match result { Err(RunnerError::InsufficientPackages { count }) => assert_eq!(count, 1), _ => panic!("Expected InsufficientPackages error"), @@ -849,8 +727,11 @@ mod tests { fs::create_dir(&pkg2_dir).expect("failed to create pkg2"); fs::write(pkg2_dir.join("Cargo.toml"), "[package]\nname = \"dup_pkg\"") .expect("failed to write Cargo.toml for pkg2"); - let runner = Runner::new(PathBuf::from("dummy_base")); - let result = runner.run_test_case(&test_case_dir); + let runner = dummy_runner(); + let test_case = TestCase::try_new(temp_dir.path(), Path::new("dup_pkg")) + .unwrap() + .unwrap(); + let result = runner.run_test_case(&test_case); match result { Err(RunnerError::DuplicatePackageName { pkg_name }) => assert_eq!(pkg_name, "dup_pkg"), _ => panic!("Expected DuplicatePackageName error"), @@ -948,7 +829,7 @@ mod tests { #[test] fn test_group_outputs_with_epsilon() { - let runner = Runner::new(PathBuf::from("dummy_base")); + let runner = dummy_runner(); // Create float outputs with small differences let val1: f32 = 1.0; diff --git a/tests/difftests/bin/src/testcase.rs b/tests/difftests/bin/src/testcase.rs new file mode 100644 index 0000000000..d758617b28 --- /dev/null +++ b/tests/difftests/bin/src/testcase.rs @@ -0,0 +1,175 @@ +use crate::runner::RunnerResult; +use std::fmt::{Display, Formatter}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tracing::debug; + +/// A test case containing multiple test binaries that should produce the same output +pub struct TestCase { + /// the relative path from the base difftest dir + pub relative_path: Arc, + /// the absolute path + pub absolute_path: PathBuf, + /// All the test binaries of this single test case. + pub test_binaries: Vec, +} + +impl TestCase { + pub fn try_new(root: &Path, relative_path: &Path) -> RunnerResult> { + let absolute_path = root.join(relative_path); + let mut test_case = TestCase { + absolute_path, + relative_path: Arc::new(relative_path.to_path_buf()), + test_binaries: Vec::new(), + }; + test_case.collect_test_binaries()?; + if test_case.test_binaries.len() > 0 { + debug!("Test case found: {}", relative_path.display()); + Ok(Some(test_case)) + } else { + Ok(None) + } + } + + fn collect_test_binaries(&mut self) -> RunnerResult<()> { + for entry in fs::read_dir(&self.absolute_path)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() && path.join("Cargo.toml").exists() { + debug!("Found binary package candidate: {}", path.display()); + self.test_binaries + .push(TestBinary::new(self, PathBuf::from(entry.file_name()))); + } + } + Ok(()) + } +} + +impl Display for TestCase { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "difftests::{}", self.relative_path.to_string_lossy()) + } +} + +/// A test binary that can be executed +pub struct TestBinary { + /// the relative path from the test case + pub relative_path: PathBuf, + /// the relative path from the test case + pub test_case_relative_path: Arc, + /// the absolute path + pub absolute_path: PathBuf, +} + +impl TestBinary { + pub fn new(test_case: &TestCase, relative_path: PathBuf) -> Self { + Self { + absolute_path: test_case.absolute_path.join(&relative_path), + test_case_relative_path: test_case.relative_path.clone(), + relative_path, + } + } + + pub fn path_from_root(&self) -> PathBuf { + self.test_case_relative_path.join(&self.relative_path) + } +} + +impl Display for TestBinary { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "difftests::{}::{}", + self.test_case_relative_path.to_string_lossy(), + self.relative_path.to_string_lossy() + ) + } +} + +pub fn collect_test_dirs(root: &Path) -> RunnerResult> { + fn recurse(root: &Path, traverse: &Path, test_cases: &mut Vec) -> RunnerResult<()> { + let absolute_path = root.join(traverse); + // skip target dir + if absolute_path.file_name() == Some(std::ffi::OsStr::new("target")) { + return Ok(()); + } + + if let Some(test_case) = TestCase::try_new(root, &traverse)? { + test_cases.push(test_case); + } + for entry in fs::read_dir(absolute_path)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + let relative_path = traverse.join(entry.file_name()); + recurse(root, &relative_path, test_cases)?; + } + } + Ok(()) + } + + let mut test_cases = Vec::new(); + recurse(root, Path::new(""), &mut test_cases)?; + Ok(test_cases) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn test_format_test_name() { + let test_case_relative_path = Arc::new(PathBuf::from("group1")); + let mut test_case = TestCase { + absolute_path: PathBuf::from("/home/user/tests/group1"), + relative_path: test_case_relative_path, + test_binaries: Vec::new(), + }; + test_case + .test_binaries + .push(TestBinary::new(&test_case, PathBuf::from("testcase1"))); + assert_eq!(test_case.to_string(), "difftests::group1"); + assert_eq!( + test_case.test_binaries[0].to_string(), + "difftests::group1::testcase1" + ); + } + + #[test] + fn test_collect_test_dirs() { + let temp_dir = tempdir().expect("failed to create temp dir"); + let base = temp_dir.path(); + let test_case_dir = base.join("test_case"); + fs::create_dir(&test_case_dir).expect("failed to create test_case dir"); + let pkg1_dir = test_case_dir.join("pkg1"); + fs::create_dir(&pkg1_dir).expect("failed to create pkg1"); + fs::write(pkg1_dir.join("Cargo.toml"), "[package]\nname = \"pkg1\"") + .expect("failed to write Cargo.toml for pkg1"); + let pkg2_dir = test_case_dir.join("pkg2"); + fs::create_dir(&pkg2_dir).expect("failed to create pkg2"); + fs::write(pkg2_dir.join("Cargo.toml"), "[package]\nname = \"pkg2\"") + .expect("failed to write Cargo.toml for pkg2"); + let mut test_dirs = collect_test_dirs(base).expect("failed to collect test dirs"); + + assert_eq!(test_dirs.len(), 1); + let test_case = &mut test_dirs[0]; + assert_eq!(test_case.relative_path.to_string_lossy(), "test_case"); + assert_eq!(test_case.absolute_path, test_case_dir); + + test_case + .test_binaries + .sort_by(|a, b| a.relative_path.cmp(&b.relative_path)); + assert_eq!( + test_case.test_binaries[0].relative_path.to_string_lossy(), + "pkg1" + ); + assert_eq!(test_case.test_binaries[0].absolute_path, pkg1_dir); + assert_eq!( + test_case.test_binaries[1].relative_path.to_string_lossy(), + "pkg2" + ); + assert_eq!(test_case.test_binaries[1].absolute_path, pkg2_dir); + } +} From dd46327f500f1eba139b62df63217b6bb775e752 Mon Sep 17 00:00:00 2001 From: Firestar99 Date: Tue, 15 Jul 2025 11:51:44 +0200 Subject: [PATCH 02/17] difftest: fix test mod paths with subdirectories --- tests/difftests/bin/src/runner.rs | 2 +- tests/difftests/bin/src/testcase.rs | 77 ++++++++++++++--------------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/tests/difftests/bin/src/runner.rs b/tests/difftests/bin/src/runner.rs index f0e4052921..dc7619c2a1 100644 --- a/tests/difftests/bin/src/runner.rs +++ b/tests/difftests/bin/src/runner.rs @@ -255,7 +255,7 @@ impl Runner { let pkg_name = self.get_package_name(&manifest_path)?; debug!("Package '{}' detected", pkg_name); - let package_out = self.output_dir.join(package.path_from_root()); + let package_out = self.output_dir.join(&package.relative_path); fs::create_dir_all(&package_out)?; debug!("Writing output to '{}'", package_out.display()); let config_file = package_out.join("config.json"); diff --git a/tests/difftests/bin/src/testcase.rs b/tests/difftests/bin/src/testcase.rs index d758617b28..dae332ade9 100644 --- a/tests/difftests/bin/src/testcase.rs +++ b/tests/difftests/bin/src/testcase.rs @@ -2,13 +2,14 @@ use crate::runner::RunnerResult; use std::fmt::{Display, Formatter}; use std::fs; use std::path::{Path, PathBuf}; -use std::sync::Arc; use tracing::debug; /// A test case containing multiple test binaries that should produce the same output pub struct TestCase { - /// the relative path from the base difftest dir - pub relative_path: Arc, + /// The name of the testcase, as a rust mod path + pub name: String, + /// the relative path from the base `difftest` dir + pub relative_path: PathBuf, /// the absolute path pub absolute_path: PathBuf, /// All the test binaries of this single test case. @@ -16,15 +17,22 @@ pub struct TestCase { } impl TestCase { - pub fn try_new(root: &Path, relative_path: &Path) -> RunnerResult> { - let absolute_path = root.join(relative_path); - let mut test_case = TestCase { - absolute_path, - relative_path: Arc::new(relative_path.to_path_buf()), + pub fn new_empty(root: &Path, relative_path: &Path) -> Self { + TestCase { + name: format!( + "difftests::{}", + relative_path.to_string_lossy().replace("/", "::") + ), + absolute_path: root.join(relative_path), + relative_path: relative_path.to_path_buf(), test_binaries: Vec::new(), - }; + } + } + + pub fn try_new(root: &Path, relative_path: &Path) -> RunnerResult> { + let mut test_case = Self::new_empty(root, relative_path); test_case.collect_test_binaries()?; - if test_case.test_binaries.len() > 0 { + if !test_case.test_binaries.is_empty() { debug!("Test case found: {}", relative_path.display()); Ok(Some(test_case)) } else { @@ -48,42 +56,37 @@ impl TestCase { impl Display for TestCase { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "difftests::{}", self.relative_path.to_string_lossy()) + f.write_str(&self.name) } } /// A test binary that can be executed pub struct TestBinary { - /// the relative path from the test case + /// The name of the testcase, as a rust mod path + pub name: String, + /// the relative path from the base `difftest` dir pub relative_path: PathBuf, - /// the relative path from the test case - pub test_case_relative_path: Arc, /// the absolute path pub absolute_path: PathBuf, } impl TestBinary { - pub fn new(test_case: &TestCase, relative_path: PathBuf) -> Self { + pub fn new(test_case: &TestCase, relative_to_test_case: PathBuf) -> Self { Self { - absolute_path: test_case.absolute_path.join(&relative_path), - test_case_relative_path: test_case.relative_path.clone(), - relative_path, + name: format!( + "{}::{}", + test_case.name, + relative_to_test_case.to_string_lossy().replace("/", "::") + ), + relative_path: test_case.relative_path.join(&relative_to_test_case), + absolute_path: test_case.absolute_path.join(&relative_to_test_case), } } - - pub fn path_from_root(&self) -> PathBuf { - self.test_case_relative_path.join(&self.relative_path) - } } impl Display for TestBinary { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "difftests::{}::{}", - self.test_case_relative_path.to_string_lossy(), - self.relative_path.to_string_lossy() - ) + f.write_str(&self.name) } } @@ -95,7 +98,7 @@ pub fn collect_test_dirs(root: &Path) -> RunnerResult> { return Ok(()); } - if let Some(test_case) = TestCase::try_new(root, &traverse)? { + if let Some(test_case) = TestCase::try_new(root, traverse)? { test_cases.push(test_case); } for entry in fs::read_dir(absolute_path)? { @@ -121,19 +124,15 @@ mod tests { #[test] fn test_format_test_name() { - let test_case_relative_path = Arc::new(PathBuf::from("group1")); - let mut test_case = TestCase { - absolute_path: PathBuf::from("/home/user/tests/group1"), - relative_path: test_case_relative_path, - test_binaries: Vec::new(), - }; + let mut test_case = + TestCase::new_empty(Path::new("/home/user/tests"), Path::new("core/group1")); test_case .test_binaries .push(TestBinary::new(&test_case, PathBuf::from("testcase1"))); - assert_eq!(test_case.to_string(), "difftests::group1"); + assert_eq!(test_case.to_string(), "difftests::core::group1"); assert_eq!( test_case.test_binaries[0].to_string(), - "difftests::group1::testcase1" + "difftests::core::group1::testcase1" ); } @@ -163,12 +162,12 @@ mod tests { .sort_by(|a, b| a.relative_path.cmp(&b.relative_path)); assert_eq!( test_case.test_binaries[0].relative_path.to_string_lossy(), - "pkg1" + "test_case/pkg1" ); assert_eq!(test_case.test_binaries[0].absolute_path, pkg1_dir); assert_eq!( test_case.test_binaries[1].relative_path.to_string_lossy(), - "pkg2" + "test_case/pkg2" ); assert_eq!(test_case.test_binaries[1].absolute_path, pkg2_dir); } From 546b586f87e73d110149707012ecd02aa60324e5 Mon Sep 17 00:00:00 2001 From: Firestar99 Date: Tue, 15 Jul 2025 13:33:56 +0200 Subject: [PATCH 03/17] difftest: fix test names on windows --- tests/difftests/bin/src/testcase.rs | 31 ++++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/difftests/bin/src/testcase.rs b/tests/difftests/bin/src/testcase.rs index dae332ade9..7ba81c8592 100644 --- a/tests/difftests/bin/src/testcase.rs +++ b/tests/difftests/bin/src/testcase.rs @@ -16,13 +16,16 @@ pub struct TestCase { pub test_binaries: Vec, } +fn path_to_test_name(path: &Path) -> String { + path.to_string_lossy() + .replace("/", "::") + .replace("\\\\", "::") +} + impl TestCase { pub fn new_empty(root: &Path, relative_path: &Path) -> Self { TestCase { - name: format!( - "difftests::{}", - relative_path.to_string_lossy().replace("/", "::") - ), + name: format!("difftests::{}", path_to_test_name(relative_path)), absolute_path: root.join(relative_path), relative_path: relative_path.to_path_buf(), test_binaries: Vec::new(), @@ -47,7 +50,7 @@ impl TestCase { if path.is_dir() && path.join("Cargo.toml").exists() { debug!("Found binary package candidate: {}", path.display()); self.test_binaries - .push(TestBinary::new(self, PathBuf::from(entry.file_name()))); + .push(TestBinary::new(self, Path::new(&entry.file_name()))); } } Ok(()) @@ -71,15 +74,15 @@ pub struct TestBinary { } impl TestBinary { - pub fn new(test_case: &TestCase, relative_to_test_case: PathBuf) -> Self { + pub fn new(test_case: &TestCase, relative_to_test_case: &Path) -> Self { Self { name: format!( "{}::{}", test_case.name, - relative_to_test_case.to_string_lossy().replace("/", "::") + path_to_test_name(relative_to_test_case) ), - relative_path: test_case.relative_path.join(&relative_to_test_case), - absolute_path: test_case.absolute_path.join(&relative_to_test_case), + relative_path: test_case.relative_path.join(relative_to_test_case), + absolute_path: test_case.absolute_path.join(relative_to_test_case), } } } @@ -128,7 +131,7 @@ mod tests { TestCase::new_empty(Path::new("/home/user/tests"), Path::new("core/group1")); test_case .test_binaries - .push(TestBinary::new(&test_case, PathBuf::from("testcase1"))); + .push(TestBinary::new(&test_case, Path::new("testcase1"))); assert_eq!(test_case.to_string(), "difftests::core::group1"); assert_eq!( test_case.test_binaries[0].to_string(), @@ -161,13 +164,13 @@ mod tests { .test_binaries .sort_by(|a, b| a.relative_path.cmp(&b.relative_path)); assert_eq!( - test_case.test_binaries[0].relative_path.to_string_lossy(), - "test_case/pkg1" + test_case.test_binaries[0].relative_path, + Path::new("test_case/pkg1") ); assert_eq!(test_case.test_binaries[0].absolute_path, pkg1_dir); assert_eq!( - test_case.test_binaries[1].relative_path.to_string_lossy(), - "test_case/pkg2" + test_case.test_binaries[1].relative_path, + Path::new("test_case/pkg2") ); assert_eq!(test_case.test_binaries[1].absolute_path, pkg2_dir); } From eb7645e0e25c4b77ca97410143bd7e581af9a8c2 Mon Sep 17 00:00:00 2001 From: Firestar99 Date: Thu, 17 Jul 2025 15:58:36 +0200 Subject: [PATCH 04/17] difftest: move difftest `bin` to `runner` --- Cargo.lock | 2 +- Cargo.toml | 4 ++-- tests/difftests/{bin => runner}/Cargo.toml | 2 +- tests/difftests/{bin => runner}/src/differ.rs | 0 tests/difftests/{bin/src/main.rs => runner/src/lib.rs} | 2 +- tests/difftests/{bin => runner}/src/runner.rs | 0 tests/difftests/{bin => runner}/src/testcase.rs | 0 7 files changed, 5 insertions(+), 5 deletions(-) rename tests/difftests/{bin => runner}/Cargo.toml (97%) rename tests/difftests/{bin => runner}/src/differ.rs (100%) rename tests/difftests/{bin/src/main.rs => runner/src/lib.rs} (99%) rename tests/difftests/{bin => runner}/src/runner.rs (100%) rename tests/difftests/{bin => runner}/src/testcase.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index fdc5a17b1c..36536b21a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -779,7 +779,7 @@ dependencies = [ ] [[package]] -name = "difftests" +name = "difftest-runner" version = "0.9.0" dependencies = [ "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 7cf0998410..de8ca77abd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ members = [ "tests/compiletests", "tests/compiletests/deps-helper", - "tests/difftests/bin", + "tests/difftests/runner", "tests/difftests/lib", ] @@ -53,7 +53,7 @@ rustc_codegen_spirv-types = { path = "./crates/rustc_codegen_spirv-types", versi rustc_codegen_spirv-target-specs = { path = "crates/rustc_codegen_spirv-target-specs", version = "=0.9.0" } # difftest libraries mirrored from difftest workspace -difftest = { path = "tests/difftests/lib" } +difftest-runner = { path = "tests/difftests/runner" } # External dependencies that need to be mentioned more than once. tracing = "0.1" diff --git a/tests/difftests/bin/Cargo.toml b/tests/difftests/runner/Cargo.toml similarity index 97% rename from tests/difftests/bin/Cargo.toml rename to tests/difftests/runner/Cargo.toml index 6eb5ca6613..a332debc06 100644 --- a/tests/difftests/bin/Cargo.toml +++ b/tests/difftests/runner/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "difftests" +name = "difftest-runner" version.workspace = true authors.workspace = true edition.workspace = true diff --git a/tests/difftests/bin/src/differ.rs b/tests/difftests/runner/src/differ.rs similarity index 100% rename from tests/difftests/bin/src/differ.rs rename to tests/difftests/runner/src/differ.rs diff --git a/tests/difftests/bin/src/main.rs b/tests/difftests/runner/src/lib.rs similarity index 99% rename from tests/difftests/bin/src/main.rs rename to tests/difftests/runner/src/lib.rs index 6856f238a4..0453a01eef 100644 --- a/tests/difftests/bin/src/main.rs +++ b/tests/difftests/runner/src/lib.rs @@ -18,7 +18,7 @@ mod differ; mod runner; mod testcase; -fn main() -> Result<()> { +fn run() -> Result<()> { let subscriber = FmtSubscriber::builder() .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .finish(); diff --git a/tests/difftests/bin/src/runner.rs b/tests/difftests/runner/src/runner.rs similarity index 100% rename from tests/difftests/bin/src/runner.rs rename to tests/difftests/runner/src/runner.rs diff --git a/tests/difftests/bin/src/testcase.rs b/tests/difftests/runner/src/testcase.rs similarity index 100% rename from tests/difftests/bin/src/testcase.rs rename to tests/difftests/runner/src/testcase.rs From 5e46e620ad2974b4f30044754e1c938989424f38 Mon Sep 17 00:00:00 2001 From: firestar99 Date: Mon, 8 Sep 2025 13:04:02 +0200 Subject: [PATCH 05/17] difftest: switch to `mimic-libtest` from `tester` --- Cargo.lock | 28 ++++++- Cargo.toml | 1 + tests/difftests/bin/Cargo.toml | 25 ++++++ tests/difftests/bin/src/test.rs | 6 ++ tests/difftests/runner/Cargo.toml | 2 +- tests/difftests/runner/src/lib.rs | 123 ++++++------------------------ 6 files changed, 83 insertions(+), 102 deletions(-) create mode 100644 tests/difftests/bin/Cargo.toml create mode 100644 tests/difftests/bin/src/test.rs diff --git a/Cargo.lock b/Cargo.lock index 36536b21a8..b555ad76c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -786,17 +786,25 @@ dependencies = [ "bytemuck", "bytesize", "difftest", + "libtest-mimic", "serde", "serde_json", "tabled", "tempfile", - "tester", "thiserror 2.0.16", "toml 0.9.5", "tracing", "tracing-subscriber", ] +[[package]] +name = "difftests" +version = "0.9.0" +dependencies = [ + "anyhow", + "difftest-runner", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -930,6 +938,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "escape8259" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" + [[package]] name = "example-runner-ash" version = "0.0.0" @@ -1748,6 +1762,18 @@ dependencies = [ "redox_syscall 0.5.17", ] +[[package]] +name = "libtest-mimic" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5297962ef19edda4ce33aaa484386e0a5b3d7f2f4e037cbeee00503ef6b29d33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" diff --git a/Cargo.toml b/Cargo.toml index de8ca77abd..30b82c4e82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ members = [ "tests/compiletests", "tests/compiletests/deps-helper", + "tests/difftests/bin", "tests/difftests/runner", "tests/difftests/lib", ] diff --git a/tests/difftests/bin/Cargo.toml b/tests/difftests/bin/Cargo.toml new file mode 100644 index 0000000000..b40435fc1f --- /dev/null +++ b/tests/difftests/bin/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "difftests" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +# See rustc_codegen_spirv/Cargo.toml for details on these features +[features] +default = ["use-compiled-tools"] +use-installed-tools = ["difftest-runner/use-installed-tools"] +use-compiled-tools = ["difftest-runner/use-compiled-tools"] + +[dependencies] +anyhow = "1.0" +difftest-runner.workspace = true + +[lints] +workspace = true + +[[test]] +name = "difftests" +path = "src/test.rs" +harness = false diff --git a/tests/difftests/bin/src/test.rs b/tests/difftests/bin/src/test.rs new file mode 100644 index 0000000000..afa31a9c9c --- /dev/null +++ b/tests/difftests/bin/src/test.rs @@ -0,0 +1,6 @@ +use anyhow::Result; +use std::process::ExitCode; + +pub fn main() -> Result { + difftest_runner::run() +} diff --git a/tests/difftests/runner/Cargo.toml b/tests/difftests/runner/Cargo.toml index a332debc06..9e69c5deac 100644 --- a/tests/difftests/runner/Cargo.toml +++ b/tests/difftests/runner/Cargo.toml @@ -16,7 +16,7 @@ use-compiled-tools = [] anyhow = "1.0" tracing = "0.1" tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"] } -tester = "0.9.1" +libtest-mimic = "0.8.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "2.0.12" diff --git a/tests/difftests/runner/src/lib.rs b/tests/difftests/runner/src/lib.rs index 0453a01eef..5482c48960 100644 --- a/tests/difftests/runner/src/lib.rs +++ b/tests/difftests/runner/src/lib.rs @@ -2,15 +2,13 @@ use crate::testcase::collect_test_dirs; use anyhow::Result; +use libtest_mimic::{Arguments, Trial}; use runner::Runner; +use std::process::ExitCode; +use std::sync::Arc; use std::{ env, fs, - process::{self, Command}, -}; -use tester::{ - ColorConfig, DynTestName, OutputFormat, RunIgnored, ShouldPanic, TestDesc, TestDescAndFn, - TestFn, TestType, run_tests_console, - test::{TestOpts, parse_opts}, + process::{self}, }; use tracing_subscriber::FmtSubscriber; @@ -18,45 +16,24 @@ mod differ; mod runner; mod testcase; -fn run() -> Result<()> { +pub fn run() -> Result { let subscriber = FmtSubscriber::builder() .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .finish(); tracing::subscriber::set_global_default(subscriber).expect("Failed to set global subscriber"); - let args: Vec = env::args().collect(); - let opts: TestOpts = match parse_opts(&args) { - Some(Ok(o)) => TestOpts { - test_threads: Some(1), - ..o - }, - Some(Err(e)) => { - eprintln!("Error parsing test options: {e}"); - process::exit(1); - } - None => TestOpts { - list: false, - filters: vec![], - filter_exact: false, - force_run_in_process: false, - exclude_should_panic: false, - run_ignored: RunIgnored::No, - run_tests: true, - bench_benchmarks: false, - logfile: None, - nocapture: false, - color: ColorConfig::AutoColor, - format: OutputFormat::Pretty, - test_threads: Some(1), - skip: vec![], - time_options: None, - options: tester::Options { - display_output: true, - panic_abort: true, - }, - }, - }; + let mut args = Arguments::from_args(); + + // If filters are provided that look like paths (contain '/'), convert them to test names + if let Some(filter) = &mut args.filter { + *filter = filter.replace('/', "::"); + } + let tests = collect_tests()?; + Ok(libtest_mimic::run(&args, tests).exit_code()) +} + +fn collect_tests() -> Result> { // Find the manifest directory at compile time and locate tests in ../tests. let manifest_dir = env!("CARGO_MANIFEST_DIR"); let base = std::path::Path::new(manifest_dir) @@ -72,10 +49,10 @@ fn run() -> Result<()> { .expect("Failed to canonicalize tests directory"); tracing::debug!("Using output directory: {}", output_dir.display()); - let runner = Runner { + let runner = Arc::new(Runner { base_dir: base.clone(), output_dir, - }; + }); let test_cases = collect_test_dirs(&base).expect("Failed to collect test case directories"); if test_cases.is_empty() { @@ -83,66 +60,12 @@ fn run() -> Result<()> { process::exit(1); } - // We build first to ensure that the tests are compiled before running them and to - // passthrough stdout and stderr from cargo to help debugging. - let mut cmd = Command::new("cargo"); - let cmd = cmd.arg("build").arg("--release"); - runner::forward_features(cmd); - cmd.current_dir(&base) - .stderr(process::Stdio::inherit()) - .stdout(process::Stdio::inherit()); - tracing::debug!("Running cargo command: {:?}", cmd); - - let output = cmd.output().expect("build output"); - let exit_code = output.status.code().unwrap_or(-1); - tracing::debug!("Cargo build exited with code {}", exit_code); - if !output.status.success() { - tracing::error!("Cargo build failed"); - process::exit(exit_code); - } - - let tests: Vec = test_cases + let trails = test_cases .into_iter() - .map(|case| TestDescAndFn { - desc: TestDesc { - name: DynTestName(case.to_string()), - ignore: false, - should_panic: ShouldPanic::No, - allow_fail: false, - test_type: TestType::IntegrationTest, - }, - testfn: TestFn::DynTestFn(Box::new({ - let runner = runner.clone(); - move || { - runner - .run_test_case(&case) - .unwrap_or_else(|e| panic!("{}", e)); - } - })), + .map(|case| { + let runner = runner.clone(); + Trial::test(case.to_string(), move || Ok(runner.run_test_case(&case)?)) }) .collect(); - - // If filters are provided that look like paths (contain '/'), convert them to test names - let opts = if opts.filters.iter().any(|f| f.contains('/')) { - let mut new_opts = opts; - new_opts.filters = new_opts - .filters - .into_iter() - .map(|filter| { - if filter.contains('/') { - // Convert path-like filter to test name format - filter.replace('/', "::") - } else { - filter - } - }) - .collect(); - new_opts - } else { - opts - }; - - let passed = run_tests_console(&opts, tests).expect("Failed to run tests"); - - process::exit(if passed { 0 } else { 1 }); + Ok(trails) } From 16c5500f26fe3e7cd372c7c80f51db969716845d Mon Sep 17 00:00:00 2001 From: Firestar99 Date: Tue, 15 Jul 2025 14:26:29 +0200 Subject: [PATCH 06/17] difftest: disable logger when listing available tests --- tests/difftests/runner/src/lib.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/difftests/runner/src/lib.rs b/tests/difftests/runner/src/lib.rs index 5482c48960..dab1cae295 100644 --- a/tests/difftests/runner/src/lib.rs +++ b/tests/difftests/runner/src/lib.rs @@ -17,13 +17,17 @@ mod runner; mod testcase; pub fn run() -> Result { - let subscriber = FmtSubscriber::builder() - .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) - .finish(); - tracing::subscriber::set_global_default(subscriber).expect("Failed to set global subscriber"); - let mut args = Arguments::from_args(); + // list must not print anything else to stdout + if !args.list { + let subscriber = FmtSubscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .finish(); + tracing::subscriber::set_global_default(subscriber) + .expect("Failed to set global subscriber"); + } + // If filters are provided that look like paths (contain '/'), convert them to test names if let Some(filter) = &mut args.filter { *filter = filter.replace('/', "::"); From 1a480f032a942827999dc11273bdf47c041c1e9b Mon Sep 17 00:00:00 2001 From: Firestar99 Date: Tue, 15 Jul 2025 14:28:57 +0200 Subject: [PATCH 07/17] difftest: fix `use-installed-tools` feature --- Cargo.toml | 2 +- tests/difftests/runner/src/runner.rs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 30b82c4e82..d6cb5bdfcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,7 @@ rustc_codegen_spirv-types = { path = "./crates/rustc_codegen_spirv-types", versi rustc_codegen_spirv-target-specs = { path = "crates/rustc_codegen_spirv-target-specs", version = "=0.9.0" } # difftest libraries mirrored from difftest workspace -difftest-runner = { path = "tests/difftests/runner" } +difftest-runner = { path = "tests/difftests/runner", default-features = false } # External dependencies that need to be mentioned more than once. tracing = "0.1" diff --git a/tests/difftests/runner/src/runner.rs b/tests/difftests/runner/src/runner.rs index dc7619c2a1..9ee731824e 100644 --- a/tests/difftests/runner/src/runner.rs +++ b/tests/difftests/runner/src/runner.rs @@ -634,6 +634,10 @@ impl Runner { pub fn forward_features(cmd: &mut Command) { cmd.arg("--features"); + #[cfg(all(feature = "use-compiled-tools", feature = "use-installed-tools"))] + compile_error!( + "Features `use-compiled-tools` and `use-installed-tools` are mutually exclusive" + ); #[cfg(feature = "use-compiled-tools")] { cmd.arg("difftest/use-compiled-tools"); From 5492747feb941622a2d5cdb4fba9f1ef5fec6ef8 Mon Sep 17 00:00:00 2001 From: Firestar99 Date: Tue, 15 Jul 2025 12:52:06 +0200 Subject: [PATCH 08/17] difftest: use nextest to properly filter tests, adjust ci --- .cargo/config.toml | 2 +- .config/nextest.toml | 9 +++++++++ .github/workflows/ci.yaml | 16 ++++++++++------ 3 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 .config/nextest.toml diff --git a/.cargo/config.toml b/.cargo/config.toml index 946679c088..5787222065 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,6 +1,6 @@ [alias] compiletest = "run --release -p compiletests --" -difftest = "run --release -p difftests --" +difftest = "nextest run --release -P difftests -p difftests" run-wasm = ["run", "--release", "-p", "run-wasm", "--"] [target.'cfg(target_arch = "wasm32")'] diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 0000000000..187d76ee36 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,9 @@ +[profile.default] +default-filter = '!package(difftest*) & !package(compiletest*) & !package(example-runner-*)' +fail-fast = false + +[profile.difftests] +default-filter = 'package(difftests)' + +[profile.difftest-runner] +default-filter = 'package(difftest-runner) | package(difftest)' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e66f0c5e30..411e916483 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,6 +42,8 @@ jobs: # figure out native target triple while we're at it - name: install rust-toolchain run: echo "TARGET=$(rustc --print host-tuple)" >> "$GITHUB_ENV" + - name: install nextest + uses: taiki-e/install-action@nextest # Fetch dependencies in a separate step to clearly show how long each part # of the testing takes - name: cargo fetch --locked @@ -50,13 +52,13 @@ jobs: # Core crates # Compiled in --release because cargo compiletest would otherwise compile in release again. - name: rustc_codegen_spirv build - run: cargo test -p rustc_codegen_spirv --release --no-default-features --features "use-installed-tools" --no-run + run: cargo nextest run -p rustc_codegen_spirv --release --no-default-features --features "use-installed-tools" --no-run - name: rustc_codegen_spirv test - run: cargo test -p rustc_codegen_spirv --release --no-default-features --features "use-installed-tools" + run: cargo nextest run -p rustc_codegen_spirv --release --no-default-features --features "use-installed-tools" - name: workspace test (excluding examples & difftest) - run: cargo test --release --workspace --exclude "example-runner-*" --exclude "difftest*" --no-default-features --features "use-installed-tools" + run: cargo nextest run --release --workspace --exclude "example-runner-*" --exclude "difftest*" --no-default-features --features "use-installed-tools" # Examples - name: cargo check examples @@ -191,14 +193,16 @@ jobs: sudo apt install -y xvfb libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers - name: install rust-toolchain run: echo "TARGET=$(rustc --print host-tuple)" >> "$GITHUB_ENV" + - name: install nextest + uses: taiki-e/install-action@nextest - name: cargo fetch --locked run: cargo fetch --locked --target $TARGET - name: cargo fetch --locked difftests run: cargo fetch --locked --manifest-path=tests/difftests/tests/Cargo.toml --target $TARGET - - name: test difftest - run: cargo test -p "difftest*" --release --no-default-features --features "use-installed-tools" + - name: test difftest-runner + run: cargo nextest run -P difftest-runner -p difftest-runner -p difftest --release --no-default-features --features "use-installed-tools" - name: difftests - run: cargo run -p difftests --release --no-default-features --features "use-installed-tools" + run: cargo nextest run -P difftests -p difftests --release --no-default-features --features "use-installed-tools" # This allows us to have a single job we can branch protect on, rather than needing # to update the branch protection rules when the test matrix changes From b612fd78dec0d3b7b5a2a8ab1bca698d94b30d38 Mon Sep 17 00:00:00 2001 From: firestar99 Date: Mon, 8 Sep 2025 13:05:21 +0200 Subject: [PATCH 09/17] difftest: move `lib` into `tests` workspace, move `mod config` to new crate `types`, massive difftest speedup 36s -> 6s --- .config/nextest.toml | 2 +- .github/workflows/ci.yaml | 2 +- Cargo.lock | 43 +- Cargo.toml | 2 +- tests/difftests/runner/Cargo.toml | 2 +- tests/difftests/runner/src/differ.rs | 2 +- tests/difftests/runner/src/runner.rs | 10 +- tests/difftests/runner/src/testcase.rs | 19 +- tests/difftests/tests/Cargo.lock | 554 +++++++++++++++++- tests/difftests/tests/Cargo.toml | 5 +- tests/difftests/{ => tests}/lib/Cargo.toml | 5 +- tests/difftests/{ => tests}/lib/src/lib.rs | 2 +- .../lib/src/scaffold/compute/ash.rs | 0 .../lib/src/scaffold/compute/backend.rs | 0 .../lib/src/scaffold/compute/mod.rs | 0 .../lib/src/scaffold/compute/wgpu.rs | 0 .../{ => tests}/lib/src/scaffold/mod.rs | 0 .../lib/src/scaffold/shader/mod.rs | 0 .../src/scaffold/shader/rust_gpu_shader.rs | 0 .../lib/src/scaffold/shader/wgsl_shader.rs | 0 .../{ => tests}/lib/src/scaffold/skip.rs | 0 tests/difftests/types/Cargo.toml | 16 + tests/difftests/{lib => types}/src/config.rs | 0 tests/difftests/types/src/lib.rs | 1 + 24 files changed, 614 insertions(+), 51 deletions(-) rename tests/difftests/{ => tests}/lib/Cargo.toml (87%) rename tests/difftests/{ => tests}/lib/src/lib.rs (97%) rename tests/difftests/{ => tests}/lib/src/scaffold/compute/ash.rs (100%) rename tests/difftests/{ => tests}/lib/src/scaffold/compute/backend.rs (100%) rename tests/difftests/{ => tests}/lib/src/scaffold/compute/mod.rs (100%) rename tests/difftests/{ => tests}/lib/src/scaffold/compute/wgpu.rs (100%) rename tests/difftests/{ => tests}/lib/src/scaffold/mod.rs (100%) rename tests/difftests/{ => tests}/lib/src/scaffold/shader/mod.rs (100%) rename tests/difftests/{ => tests}/lib/src/scaffold/shader/rust_gpu_shader.rs (100%) rename tests/difftests/{ => tests}/lib/src/scaffold/shader/wgsl_shader.rs (100%) rename tests/difftests/{ => tests}/lib/src/scaffold/skip.rs (100%) create mode 100644 tests/difftests/types/Cargo.toml rename tests/difftests/{lib => types}/src/config.rs (100%) create mode 100644 tests/difftests/types/src/lib.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 187d76ee36..31d6c0d120 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -6,4 +6,4 @@ fail-fast = false default-filter = 'package(difftests)' [profile.difftest-runner] -default-filter = 'package(difftest-runner) | package(difftest)' +default-filter = 'package(difftest-runner) | package(difftest-types)' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 411e916483..9eec4235fb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -200,7 +200,7 @@ jobs: - name: cargo fetch --locked difftests run: cargo fetch --locked --manifest-path=tests/difftests/tests/Cargo.toml --target $TARGET - name: test difftest-runner - run: cargo nextest run -P difftest-runner -p difftest-runner -p difftest --release --no-default-features --features "use-installed-tools" + run: cargo nextest run -P difftest-runner -p difftest-runner -p difftest-types --release --no-default-features --features "use-installed-tools" - name: difftests run: cargo nextest run -P difftests -p difftests --release --no-default-features --features "use-installed-tools" diff --git a/Cargo.lock b/Cargo.lock index b555ad76c5..d0caa6ba78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -763,38 +763,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" [[package]] -name = "difftest" +name = "difftest-runner" version = "0.9.0" dependencies = [ "anyhow", - "ash", "bytemuck", - "futures", - "naga", + "bytesize", + "difftest-types", + "libtest-mimic", "serde", "serde_json", - "spirv-builder", + "tabled", "tempfile", - "wgpu", + "thiserror 2.0.16", + "toml 0.9.5", + "tracing", + "tracing-subscriber", ] [[package]] -name = "difftest-runner" +name = "difftest-types" version = "0.9.0" dependencies = [ "anyhow", "bytemuck", - "bytesize", - "difftest", - "libtest-mimic", "serde", "serde_json", - "tabled", - "tempfile", - "thiserror 2.0.16", - "toml 0.9.5", - "tracing", - "tracing-subscriber", ] [[package]] @@ -1820,12 +1814,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3bd0dd2cd90571056fdb71f6275fada10131182f84899f4b2a916e565d81d86" -[[package]] -name = "mach-dxcompiler-rs" -version = "0.1.4+2024.11.22-df583a3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3cd67e8ea2ba061339150970542cf1c60ba44c6d17e31279cbc133a4b018f8" - [[package]] name = "malloc_buf" version = "0.0.6" @@ -1978,7 +1966,6 @@ dependencies = [ "num-traits", "once_cell", "petgraph", - "pp-rs", "rustc-hash", "spirv", "thiserror 2.0.16", @@ -2557,15 +2544,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "pp-rs" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb458bb7f6e250e6eb79d5026badc10a3ebb8f9a15d1fff0f13d17c71f4d6dee" -dependencies = [ - "unicode-xid", -] - [[package]] name = "presser" version = "0.3.1" @@ -4247,7 +4225,6 @@ dependencies = [ "libc", "libloading", "log", - "mach-dxcompiler-rs", "metal", "naga", "ndk-sys", diff --git a/Cargo.toml b/Cargo.toml index d6cb5bdfcb..2d6a55ad0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ members = [ "tests/compiletests/deps-helper", "tests/difftests/bin", "tests/difftests/runner", - "tests/difftests/lib", + "tests/difftests/types", ] [workspace.package] diff --git a/tests/difftests/runner/Cargo.toml b/tests/difftests/runner/Cargo.toml index 9e69c5deac..bc47ac8ae3 100644 --- a/tests/difftests/runner/Cargo.toml +++ b/tests/difftests/runner/Cargo.toml @@ -23,7 +23,7 @@ thiserror = "2.0.12" toml = { version = "0.9.2", default-features = false, features = ["parse", "serde"] } bytesize = "2.0.1" bytemuck = "1.21.0" -difftest = { path = "../lib" } +difftest-types = { path = "../types" } tabled = { version = "0.20.0", default-features = false, features = ["std"] } [dev-dependencies] diff --git a/tests/difftests/runner/src/differ.rs b/tests/difftests/runner/src/differ.rs index 62de8983f9..592046dd17 100644 --- a/tests/difftests/runner/src/differ.rs +++ b/tests/difftests/runner/src/differ.rs @@ -1,6 +1,6 @@ #![allow(clippy::unimplemented)] -use difftest::config::OutputType; +use difftest_types::config::OutputType; use std::marker::PhantomData; /// Represents the magnitude of a difference between two values diff --git a/tests/difftests/runner/src/runner.rs b/tests/difftests/runner/src/runner.rs index 9ee731824e..e1cab4bc00 100644 --- a/tests/difftests/runner/src/runner.rs +++ b/tests/difftests/runner/src/runner.rs @@ -1,6 +1,7 @@ use bytesize::ByteSize; -use difftest::config::{OutputType, TestMetadata}; +use difftest_types::config::{OutputType, TestMetadata}; use serde::{Deserialize, Serialize}; +use std::process::Stdio; use std::{ collections::{HashMap, HashSet}, fs, @@ -288,6 +289,8 @@ impl Runner { let output = cmd .current_dir(&package.absolute_path) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) .output() .map_err(|e| RunnerError::Io { source: e })?; let exit_code = output.status.code().unwrap_or(-1); @@ -651,7 +654,7 @@ pub fn forward_features(cmd: &mut Command) { #[cfg(test)] mod tests { use super::*; - use difftest::config::OutputType; + use difftest_types::config::OutputType; use std::{fs, io::Write, path::Path, path::PathBuf}; use tempfile::{NamedTempFile, tempdir}; @@ -862,8 +865,7 @@ mod tests { fn test_invalid_metadata_json() { // Test that invalid JSON in metadata file causes proper error let metadata_content = "{ invalid json }"; - let result: Result = - serde_json::from_str(metadata_content); + let result: Result = serde_json::from_str(metadata_content); assert!(result.is_err()); // Just check that it's an error, don't check the specific message } diff --git a/tests/difftests/runner/src/testcase.rs b/tests/difftests/runner/src/testcase.rs index 7ba81c8592..9715abe25b 100644 --- a/tests/difftests/runner/src/testcase.rs +++ b/tests/difftests/runner/src/testcase.rs @@ -1,4 +1,5 @@ use crate::runner::RunnerResult; +use std::ffi::OsStr; use std::fmt::{Display, Formatter}; use std::fs; use std::path::{Path, PathBuf}; @@ -47,6 +48,14 @@ impl TestCase { for entry in fs::read_dir(&self.absolute_path)? { let entry = entry?; let path = entry.path(); + let relative_path = self.relative_path.join(entry.file_name()); + if IGNORE_DIR_LIST + .iter() + .any(|dir| relative_path.as_os_str() == OsStr::new(dir)) + { + debug!("Ignoring test binary: {}", path.display()); + return Ok(()); + } if path.is_dir() && path.join("Cargo.toml").exists() { debug!("Found binary package candidate: {}", path.display()); self.test_binaries @@ -93,11 +102,17 @@ impl Display for TestBinary { } } +/// List of paths relative to `./tests/difftests/tests/` to ignore +pub const IGNORE_DIR_LIST: &[&str] = &["target", "lib"]; + pub fn collect_test_dirs(root: &Path) -> RunnerResult> { fn recurse(root: &Path, traverse: &Path, test_cases: &mut Vec) -> RunnerResult<()> { let absolute_path = root.join(traverse); - // skip target dir - if absolute_path.file_name() == Some(std::ffi::OsStr::new("target")) { + if IGNORE_DIR_LIST + .iter() + .any(|dir| traverse.as_os_str() == OsStr::new(dir)) + { + debug!("Ignoring path: {}", absolute_path.display()); return Ok(()); } diff --git a/tests/difftests/tests/Cargo.lock b/tests/difftests/tests/Cargo.lock index 74d87cce66..2a46902d49 100644 --- a/tests/difftests/tests/Cargo.lock +++ b/tests/difftests/tests/Cargo.lock @@ -2,6 +2,34 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -23,6 +51,12 @@ version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +[[package]] +name = "ar" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d67af77d68a931ecd5cbd8a3b5987d63a1d1d1278f7f6a60ae33db485cdebb69" + [[package]] name = "array_access-rust" version = "0.0.0" @@ -220,6 +254,18 @@ dependencies = [ "thiserror 2.0.16", ] +[[package]] +name = "cc" +version = "1.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.3" @@ -330,6 +376,12 @@ dependencies = [ "difftest", ] +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "core-foundation" version = "0.10.1" @@ -357,19 +409,42 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crunchy" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + [[package]] name = "difftest" -version = "0.9.0" +version = "0.0.0" dependencies = [ "anyhow", "ash", "bytemuck", + "difftest-types", "futures", "naga", "serde", @@ -379,6 +454,16 @@ dependencies = [ "wgpu", ] +[[package]] +name = "difftest-types" +version = "0.9.0" +dependencies = [ + "anyhow", + "bytemuck", + "serde", + "serde_json", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -399,6 +484,22 @@ dependencies = [ "litrs", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "elsa" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9abf33c656a7256451ebb7d0082c5a471820c31269e49d807c538c252352186e" +dependencies = [ + "indexmap", + "stable_deref_trait", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -425,18 +526,40 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "find-msvc-tools" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" + [[package]] name = "fixedbitset" version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flate2" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "foldhash" version = "0.1.5" @@ -580,6 +703,17 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +dependencies = [ + "fallible-iterator", + "indexmap", + "stable_deref_trait", +] + [[package]] name = "gl_generator" version = "0.14.0" @@ -817,6 +951,30 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "internal-iterator" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "969ee3fc68ec2e88eb21434ce4d9b7e1600d1ce92ff974560a6c4a304f5124b9" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -829,6 +987,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + [[package]] name = "js-sys" version = "0.3.78" @@ -856,6 +1024,12 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.175" @@ -912,6 +1086,12 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "longest-increasing-subsequence" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3bd0dd2cd90571056fdb71f6275fada10131182f84899f4b2a916e565d81d86" + [[package]] name = "mach-dxcompiler-rs" version = "0.1.4+2024.11.22-df583a3.1" @@ -927,6 +1107,15 @@ dependencies = [ "libc", ] +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "math_ops-rust" version = "0.0.0" @@ -997,6 +1186,15 @@ dependencies = [ "paste", ] +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + [[package]] name = "naga" version = "26.0.0" @@ -1034,6 +1232,15 @@ dependencies = [ "jni-sys", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1074,6 +1281,33 @@ dependencies = [ "malloc_buf", ] +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "crc32fast", + "flate2", + "hashbrown", + "indexmap", + "memchr", + "ruzstd", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "crc32fast", + "hashbrown", + "indexmap", + "memchr", + "wasmparser", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -1275,6 +1509,35 @@ dependencies = [ "bitflags 2.9.4", ] +[[package]] +name = "regex" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" + [[package]] name = "renderdoc-sys" version = "1.1.0" @@ -1291,12 +1554,52 @@ dependencies = [ "spirv", ] +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + [[package]] name = "rustc-hash" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_codegen_spirv" +version = "0.9.0" +dependencies = [ + "ahash", + "ar", + "bytemuck", + "either", + "indexmap", + "itertools 0.14.0", + "lazy_static", + "libc", + "log", + "object 0.37.3", + "regex", + "rspirv", + "rustc-demangle", + "rustc_codegen_spirv-target-specs", + "rustc_codegen_spirv-types", + "rustix", + "sanitize-filename", + "smallvec", + "spirt", + "spirv-tools", + "thorin-dwp", + "tracing", + "tracing-subscriber", + "tracing-tree", +] + +[[package]] +name = "rustc_codegen_spirv-target-specs" +version = "0.9.0" + [[package]] name = "rustc_codegen_spirv-types" version = "0.9.0" @@ -1307,6 +1610,15 @@ dependencies = [ "spirv", ] +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.0.8" @@ -1326,12 +1638,30 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ruzstd" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad02996bfc73da3e301efe90b1837be9ed8f4a462b6ed410aa35d00381de89f" +dependencies = [ + "twox-hash", +] + [[package]] name = "ryu" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "sanitize-filename" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc984f4f9ceb736a7bb755c3e3bd17dc56370af2600c9780dcc48c66453da34d" +dependencies = [ + "regex", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1409,6 +1739,21 @@ dependencies = [ "serde", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "simple-compute-rust" version = "0.0.0" @@ -1444,6 +1789,30 @@ name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "spirt" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2d5968bd2a36466468aac637b355776f080edfb0c6f769b2b99b9708260c42a" +dependencies = [ + "arrayvec", + "bytemuck", + "derive_more", + "elsa", + "indexmap", + "internal-iterator", + "itertools 0.10.5", + "lazy_static", + "longest-increasing-subsequence", + "rustc-hash", + "serde", + "serde_json", + "smallvec", +] [[package]] name = "spirv" @@ -1463,6 +1832,8 @@ dependencies = [ "log", "memchr", "raw-string", + "rustc_codegen_spirv", + "rustc_codegen_spirv-target-specs", "rustc_codegen_spirv-types", "semver", "serde", @@ -1496,6 +1867,26 @@ dependencies = [ name = "spirv-std-types" version = "0.9.0" +[[package]] +name = "spirv-tools" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b10991c25e3adc32c018c528be2cdef16c0cdfaf8581fbeabb033b547a599ec" +dependencies = [ + "memchr", + "spirv-tools-sys", + "tempfile", +] + +[[package]] +name = "spirv-tools-sys" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c78c1a8af381edef1b7ca7d7caea9d6454dccb4b00a35b7b968b44b8212f487" +dependencies = [ + "cc", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -1592,6 +1983,27 @@ dependencies = [ "syn", ] +[[package]] +name = "thorin-dwp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9c1e705f82a260173f3eec93f2ff6d7807f23ad5a8cc2e7316a891733ea7a1" +dependencies = [ + "gimli", + "hashbrown", + "object 0.36.7", + "tracing", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "tinystr" version = "0.8.1" @@ -1643,6 +2055,92 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "tracing-tree" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f459ca79f1b0d5f71c54ddfde6debfc59c8b6eeb46808ae492077f739dc7b49c" +dependencies = [ + "nu-ansi-term", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "trig_ops-rust" version = "0.0.0" @@ -1658,6 +2156,16 @@ dependencies = [ "difftest", ] +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + [[package]] name = "typeid" version = "1.0.3" @@ -1700,6 +2208,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vector_extract_insert-rust" version = "0.0.0" @@ -1836,6 +2350,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasmparser" +version = "0.236.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9b1e81f3eb254cf7404a82cee6926a4a3ccc5aad80cc3d43608a070c67aa1d7" +dependencies = [ + "bitflags 2.9.4", +] + [[package]] name = "web-sys" version = "0.3.78" @@ -2082,6 +2605,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.60.2" @@ -2306,6 +2838,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.6" diff --git a/tests/difftests/tests/Cargo.toml b/tests/difftests/tests/Cargo.toml index 7c74495325..4bb861ffea 100644 --- a/tests/difftests/tests/Cargo.toml +++ b/tests/difftests/tests/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" members = [ + "lib", "simple-compute/simple-compute-rust", "simple-compute/simple-compute-wgsl", "arch/atomic_ops/atomic_ops-rust", @@ -53,8 +54,10 @@ unexpected_cfgs = { level = "allow", check-cfg = [ ] } [workspace.dependencies] +spirv-builder = { path = "../../../crates/spirv-builder", default-features = false } spirv-std = { path = "../../../crates/spirv-std", version = "=0.9.0" } -difftest = { path = "../../../tests/difftests/lib" } +difftest = { path = "lib" } +difftest-types = { path = "../types" } # External dependencies that need to be mentioned more than once. num-traits = { version = "0.2.15", default-features = false } glam = { version = ">=0.22, <=0.30", default-features = false } diff --git a/tests/difftests/lib/Cargo.toml b/tests/difftests/tests/lib/Cargo.toml similarity index 87% rename from tests/difftests/lib/Cargo.toml rename to tests/difftests/tests/lib/Cargo.toml index 865a329301..0a825a1207 100644 --- a/tests/difftests/lib/Cargo.toml +++ b/tests/difftests/tests/lib/Cargo.toml @@ -1,10 +1,6 @@ [package] name = "difftest" -version.workspace = true -authors.workspace = true edition.workspace = true -license.workspace = true -repository.workspace = true # See rustc_codegen_spirv/Cargo.toml for details on these features [features] @@ -17,6 +13,7 @@ use-compiled-tools = [ [target.'cfg(not(target_arch = "spirv"))'.dependencies] spirv-builder.workspace = true +difftest-types.workspace = true serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" wgpu = { version = "26.0.1", features = ["spirv", "vulkan-portability", "static-dxc"] } diff --git a/tests/difftests/lib/src/lib.rs b/tests/difftests/tests/lib/src/lib.rs similarity index 97% rename from tests/difftests/lib/src/lib.rs rename to tests/difftests/tests/lib/src/lib.rs index 2f66926766..7491ec61f0 100644 --- a/tests/difftests/lib/src/lib.rs +++ b/tests/difftests/tests/lib/src/lib.rs @@ -1,7 +1,7 @@ #![cfg_attr(target_arch = "spirv", no_std)] #[cfg(not(target_arch = "spirv"))] -pub mod config; +pub use difftest_types::config; #[cfg(not(target_arch = "spirv"))] pub mod scaffold; diff --git a/tests/difftests/lib/src/scaffold/compute/ash.rs b/tests/difftests/tests/lib/src/scaffold/compute/ash.rs similarity index 100% rename from tests/difftests/lib/src/scaffold/compute/ash.rs rename to tests/difftests/tests/lib/src/scaffold/compute/ash.rs diff --git a/tests/difftests/lib/src/scaffold/compute/backend.rs b/tests/difftests/tests/lib/src/scaffold/compute/backend.rs similarity index 100% rename from tests/difftests/lib/src/scaffold/compute/backend.rs rename to tests/difftests/tests/lib/src/scaffold/compute/backend.rs diff --git a/tests/difftests/lib/src/scaffold/compute/mod.rs b/tests/difftests/tests/lib/src/scaffold/compute/mod.rs similarity index 100% rename from tests/difftests/lib/src/scaffold/compute/mod.rs rename to tests/difftests/tests/lib/src/scaffold/compute/mod.rs diff --git a/tests/difftests/lib/src/scaffold/compute/wgpu.rs b/tests/difftests/tests/lib/src/scaffold/compute/wgpu.rs similarity index 100% rename from tests/difftests/lib/src/scaffold/compute/wgpu.rs rename to tests/difftests/tests/lib/src/scaffold/compute/wgpu.rs diff --git a/tests/difftests/lib/src/scaffold/mod.rs b/tests/difftests/tests/lib/src/scaffold/mod.rs similarity index 100% rename from tests/difftests/lib/src/scaffold/mod.rs rename to tests/difftests/tests/lib/src/scaffold/mod.rs diff --git a/tests/difftests/lib/src/scaffold/shader/mod.rs b/tests/difftests/tests/lib/src/scaffold/shader/mod.rs similarity index 100% rename from tests/difftests/lib/src/scaffold/shader/mod.rs rename to tests/difftests/tests/lib/src/scaffold/shader/mod.rs diff --git a/tests/difftests/lib/src/scaffold/shader/rust_gpu_shader.rs b/tests/difftests/tests/lib/src/scaffold/shader/rust_gpu_shader.rs similarity index 100% rename from tests/difftests/lib/src/scaffold/shader/rust_gpu_shader.rs rename to tests/difftests/tests/lib/src/scaffold/shader/rust_gpu_shader.rs diff --git a/tests/difftests/lib/src/scaffold/shader/wgsl_shader.rs b/tests/difftests/tests/lib/src/scaffold/shader/wgsl_shader.rs similarity index 100% rename from tests/difftests/lib/src/scaffold/shader/wgsl_shader.rs rename to tests/difftests/tests/lib/src/scaffold/shader/wgsl_shader.rs diff --git a/tests/difftests/lib/src/scaffold/skip.rs b/tests/difftests/tests/lib/src/scaffold/skip.rs similarity index 100% rename from tests/difftests/lib/src/scaffold/skip.rs rename to tests/difftests/tests/lib/src/scaffold/skip.rs diff --git a/tests/difftests/types/Cargo.toml b/tests/difftests/types/Cargo.toml new file mode 100644 index 0000000000..25847165e6 --- /dev/null +++ b/tests/difftests/types/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "difftest-types" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow = "1.0" +bytemuck = "1.21.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" diff --git a/tests/difftests/lib/src/config.rs b/tests/difftests/types/src/config.rs similarity index 100% rename from tests/difftests/lib/src/config.rs rename to tests/difftests/types/src/config.rs diff --git a/tests/difftests/types/src/lib.rs b/tests/difftests/types/src/lib.rs new file mode 100644 index 0000000000..ef68c36943 --- /dev/null +++ b/tests/difftests/types/src/lib.rs @@ -0,0 +1 @@ +pub mod config; From 50af67dc9b8217cb3d4d8a681d33416d9b37349b Mon Sep 17 00:00:00 2001 From: Firestar99 Date: Tue, 15 Jul 2025 21:30:46 +0200 Subject: [PATCH 10/17] difftest: compile difftest in ci before running --- .config/nextest.toml | 1 + .github/workflows/ci.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.config/nextest.toml b/.config/nextest.toml index 31d6c0d120..4edcc1c7f3 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -4,6 +4,7 @@ fail-fast = false [profile.difftests] default-filter = 'package(difftests)' +slow-timeout = "2m" [profile.difftest-runner] default-filter = 'package(difftest-runner) | package(difftest-types)' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9eec4235fb..896923cb66 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -201,6 +201,8 @@ jobs: run: cargo fetch --locked --manifest-path=tests/difftests/tests/Cargo.toml --target $TARGET - name: test difftest-runner run: cargo nextest run -P difftest-runner -p difftest-runner -p difftest-types --release --no-default-features --features "use-installed-tools" + - name: build difftests (without shaders) + run: cargo nextest run -P difftests -p difftests --release --no-default-features --features "use-installed-tools" --no-run - name: difftests run: cargo nextest run -P difftests -p difftests --release --no-default-features --features "use-installed-tools" From a18042cb459723af91e3c3652c183d650c61bee2 Mon Sep 17 00:00:00 2001 From: Firestar99 Date: Tue, 15 Jul 2025 21:53:30 +0200 Subject: [PATCH 11/17] ci: do not fail-fast matrices --- .github/workflows/ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 896923cb66..b77af74fc5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,6 +16,7 @@ jobs: test: name: Test strategy: + fail-fast: false matrix: os: [ ubuntu-24.04, windows-2022, macOS-latest ] runs-on: ${{ matrix.os }} @@ -133,6 +134,7 @@ jobs: compiletest: name: Compiletest strategy: + fail-fast: false matrix: os: [ ubuntu-24.04, windows-2022, macOS-latest ] runs-on: ${{ matrix.os }} @@ -159,6 +161,7 @@ jobs: difftest: name: Difftest strategy: + fail-fast: false matrix: os: [ ubuntu-24.04, windows-2022, macOS-latest ] runs-on: ${{ matrix.os }} From fed698fde1b064888db901fa0ecfc25e576206a2 Mon Sep 17 00:00:00 2001 From: Firestar99 Date: Tue, 15 Jul 2025 21:59:41 +0200 Subject: [PATCH 12/17] difftest: better io error reporting in config --- tests/difftests/types/src/config.rs | 36 ++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/tests/difftests/types/src/config.rs b/tests/difftests/types/src/config.rs index 8107104668..78cfdd3dae 100644 --- a/tests/difftests/types/src/config.rs +++ b/tests/difftests/types/src/config.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use serde::{Deserialize, Serialize}; use std::fs::File; use std::io::Write; @@ -11,8 +12,18 @@ pub struct Config { impl Config { pub fn write_result(&self, output: &[A]) -> anyhow::Result<()> { - let mut f = File::create(&self.output_path)?; - f.write_all(bytemuck::cast_slice(output))?; + let mut f = File::create(&self.output_path).with_context(|| { + format!( + "failed to create output file '{}'", + self.output_path.display() + ) + })?; + f.write_all(bytemuck::cast_slice(output)).with_context(|| { + format!( + "failed to write to output file '{}'", + self.output_path.display() + ) + })?; Ok(()) } } @@ -106,15 +117,28 @@ impl TestMetadata { impl Config { pub fn from_path>(path: P) -> anyhow::Result { - let content = fs::read_to_string(path)?; - let config = serde_json::from_str(&content)?; + let path = path.as_ref(); + let content = fs::read_to_string(path) + .with_context(|| format!("Could not read file '{}'", path.display()))?; + let config = serde_json::from_str(&content).with_context(|| { + format!( + "Could not parse json in file '{}':\n{content}", + path.display() + ) + })?; Ok(config) } /// Write test metadata to the configured metadata path pub fn write_metadata(&self, metadata: &TestMetadata) -> anyhow::Result<()> { - let metadata_json = serde_json::to_string(metadata)?; - fs::write(&self.metadata_path, metadata_json)?; + let metadata_json = + serde_json::to_string(metadata).context("Could not serialize TestMetadata")?; + fs::write(&self.metadata_path, metadata_json).with_context(|| { + format!( + "Could not write TestMetadata to file at '{}'", + self.metadata_path.display() + ) + })?; Ok(()) } } From d3bebba398a2a3a5b8eaf4ed545aae280cb2e381 Mon Sep 17 00:00:00 2001 From: firestar99 Date: Mon, 8 Sep 2025 13:05:58 +0200 Subject: [PATCH 13/17] difftest: cargo windows bug workaround, mirror all workspace dependencies --- Cargo.toml | 2 ++ crates/rustc_codegen_spirv/Cargo.toml | 2 +- crates/spirv-std/Cargo.toml | 2 +- tests/difftests/tests/Cargo.toml | 5 ++++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2d6a55ad0b..a12726b9ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,9 @@ rustc_codegen_spirv-types = { path = "./crates/rustc_codegen_spirv-types", versi rustc_codegen_spirv-target-specs = { path = "crates/rustc_codegen_spirv-target-specs", version = "=0.9.0" } # difftest libraries mirrored from difftest workspace +difftest = { path = "tests/difftests/tests/lib" } difftest-runner = { path = "tests/difftests/runner", default-features = false } +difftest-types = { path = "tests/difftests/types" } # External dependencies that need to be mentioned more than once. tracing = "0.1" diff --git a/crates/rustc_codegen_spirv/Cargo.toml b/crates/rustc_codegen_spirv/Cargo.toml index e08812d926..798ae60ebf 100644 --- a/crates/rustc_codegen_spirv/Cargo.toml +++ b/crates/rustc_codegen_spirv/Cargo.toml @@ -34,7 +34,7 @@ skip-toolchain-check = [] # HACK(eddyb) these only exist to unify features across dependency trees, # in order to avoid multiple separate instances of `rustc_codegen_spirv`. ahash = { version = "0.8.11", features = ["no-rng"] } -bytemuck = { version = "1.20.0", features = ["aarch64_simd", "derive"] } +bytemuck = { workspace = true, features = ["aarch64_simd"] } log = { version = "0.4.22", features = ["std"] } regex = { version = "1", features = ["perf"] } rustix = { version = "1.0.8", features = ["all-apis"] } diff --git a/crates/spirv-std/Cargo.toml b/crates/spirv-std/Cargo.toml index e5d69a08e3..c847a2edf4 100644 --- a/crates/spirv-std/Cargo.toml +++ b/crates/spirv-std/Cargo.toml @@ -14,7 +14,7 @@ workspace = true spirv-std-types.workspace = true spirv-std-macros.workspace = true bitflags = "1.3.2" -bytemuck = { version = "1.18.0", features = ["derive"], optional = true } +bytemuck = { workspace = true, optional = true } [target.'cfg(target_arch = "spirv")'.dependencies] num-traits = { workspace = true, features = ["libm"] } diff --git a/tests/difftests/tests/Cargo.toml b/tests/difftests/tests/Cargo.toml index 4bb861ffea..aaa7003d77 100644 --- a/tests/difftests/tests/Cargo.toml +++ b/tests/difftests/tests/Cargo.toml @@ -53,6 +53,9 @@ unexpected_cfgs = { level = "allow", check-cfg = [ 'cfg(target_arch, values("spirv"))' ] } +# Cargo Windows bug: workspace dependencies of crates.io dependencies are resolved +# incorrectly in the root workspace instead of this difftest workspace. So all +# workspace dependencies here must be mirrored into the root workspace as well. [workspace.dependencies] spirv-builder = { path = "../../../crates/spirv-builder", default-features = false } spirv-std = { path = "../../../crates/spirv-std", version = "=0.9.0" } @@ -61,7 +64,7 @@ difftest-types = { path = "../types" } # External dependencies that need to be mentioned more than once. num-traits = { version = "0.2.15", default-features = false } glam = { version = ">=0.22, <=0.30", default-features = false } -bytemuck = { version = "1.14", features = ["derive"] } +bytemuck = { version = "1.23", features = ["derive"] } # Enable incremental by default in release mode. [profile.release] From 743e7746e2d5dc6b2d90c191335515877030ddf4 Mon Sep 17 00:00:00 2001 From: Christian Legnitto Date: Wed, 16 Jul 2025 14:44:03 +0300 Subject: [PATCH 14/17] difftest: Unconditionally create output file --- tests/difftests/runner/src/runner.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/difftests/runner/src/runner.rs b/tests/difftests/runner/src/runner.rs index e1cab4bc00..917822b255 100644 --- a/tests/difftests/runner/src/runner.rs +++ b/tests/difftests/runner/src/runner.rs @@ -272,6 +272,9 @@ impl Runner { fs::write(&config_file, &config_json)?; trace!("Config file created at {}", config_file.display()); + fs::write(&temp_output_path, [])?; + trace!("Output file created at {}", temp_output_path.display()); + let mut cmd = Command::new("cargo"); cmd.arg("run").arg("--release").arg("--manifest-path").arg( manifest_path From 54741464dba9ca6171c87a8e835069dea7e3240e Mon Sep 17 00:00:00 2001 From: firestar99 Date: Thu, 11 Sep 2025 12:12:01 +0200 Subject: [PATCH 15/17] difftest: remove glam from workspace, use reeexport via spirv-std --- tests/difftests/tests/Cargo.lock | 2 -- tests/difftests/tests/Cargo.toml | 2 -- .../vector_extract_insert/vector_extract_insert-rust/Cargo.toml | 1 - .../vector_extract_insert/vector_extract_insert-rust/src/lib.rs | 2 +- .../lang/core/ops/vector_swizzle/vector_swizzle-rust/Cargo.toml | 1 - .../lang/core/ops/vector_swizzle/vector_swizzle-rust/src/lib.rs | 2 +- 6 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/difftests/tests/Cargo.lock b/tests/difftests/tests/Cargo.lock index 2a46902d49..c5ef3d9086 100644 --- a/tests/difftests/tests/Cargo.lock +++ b/tests/difftests/tests/Cargo.lock @@ -2219,7 +2219,6 @@ name = "vector_extract_insert-rust" version = "0.0.0" dependencies = [ "difftest", - "glam", "spirv-std", ] @@ -2252,7 +2251,6 @@ name = "vector_swizzle-rust" version = "0.0.0" dependencies = [ "difftest", - "glam", "spirv-std", ] diff --git a/tests/difftests/tests/Cargo.toml b/tests/difftests/tests/Cargo.toml index aaa7003d77..d3e9fd6e3a 100644 --- a/tests/difftests/tests/Cargo.toml +++ b/tests/difftests/tests/Cargo.toml @@ -62,8 +62,6 @@ spirv-std = { path = "../../../crates/spirv-std", version = "=0.9.0" } difftest = { path = "lib" } difftest-types = { path = "../types" } # External dependencies that need to be mentioned more than once. -num-traits = { version = "0.2.15", default-features = false } -glam = { version = ">=0.22, <=0.30", default-features = false } bytemuck = { version = "1.23", features = ["derive"] } # Enable incremental by default in release mode. diff --git a/tests/difftests/tests/arch/vector_extract_insert/vector_extract_insert-rust/Cargo.toml b/tests/difftests/tests/arch/vector_extract_insert/vector_extract_insert-rust/Cargo.toml index 2481bd472d..ea5fa2c628 100644 --- a/tests/difftests/tests/arch/vector_extract_insert/vector_extract_insert-rust/Cargo.toml +++ b/tests/difftests/tests/arch/vector_extract_insert/vector_extract_insert-rust/Cargo.toml @@ -11,7 +11,6 @@ crate-type = ["dylib"] # GPU deps [dependencies] spirv-std.workspace = true -glam.workspace = true # CPU deps (for the test harness) [target.'cfg(not(target_arch = "spirv"))'.dependencies] diff --git a/tests/difftests/tests/arch/vector_extract_insert/vector_extract_insert-rust/src/lib.rs b/tests/difftests/tests/arch/vector_extract_insert/vector_extract_insert-rust/src/lib.rs index 078f868600..4d6912e3fb 100644 --- a/tests/difftests/tests/arch/vector_extract_insert/vector_extract_insert-rust/src/lib.rs +++ b/tests/difftests/tests/arch/vector_extract_insert/vector_extract_insert-rust/src/lib.rs @@ -1,8 +1,8 @@ #![no_std] #![cfg_attr(target_arch = "spirv", feature(asm_experimental_arch))] -use glam::Vec4; use spirv_std::arch::{vector_extract_dynamic, vector_insert_dynamic}; +use spirv_std::glam::Vec4; use spirv_std::spirv; #[spirv(compute(threads(64)))] diff --git a/tests/difftests/tests/lang/core/ops/vector_swizzle/vector_swizzle-rust/Cargo.toml b/tests/difftests/tests/lang/core/ops/vector_swizzle/vector_swizzle-rust/Cargo.toml index 6cf5f5e2a3..19d900d2e5 100644 --- a/tests/difftests/tests/lang/core/ops/vector_swizzle/vector_swizzle-rust/Cargo.toml +++ b/tests/difftests/tests/lang/core/ops/vector_swizzle/vector_swizzle-rust/Cargo.toml @@ -11,7 +11,6 @@ crate-type = ["dylib"] # GPU deps [dependencies] spirv-std.workspace = true -glam.workspace = true # CPU deps (for the test harness) [target.'cfg(not(target_arch = "spirv"))'.dependencies] diff --git a/tests/difftests/tests/lang/core/ops/vector_swizzle/vector_swizzle-rust/src/lib.rs b/tests/difftests/tests/lang/core/ops/vector_swizzle/vector_swizzle-rust/src/lib.rs index acd1a7e9e8..9f9850b7f5 100644 --- a/tests/difftests/tests/lang/core/ops/vector_swizzle/vector_swizzle-rust/src/lib.rs +++ b/tests/difftests/tests/lang/core/ops/vector_swizzle/vector_swizzle-rust/src/lib.rs @@ -1,6 +1,6 @@ #![no_std] -use glam::{Vec3Swizzles, Vec4, Vec4Swizzles}; +use spirv_std::glam::{Vec3Swizzles, Vec4, Vec4Swizzles}; use spirv_std::spirv; #[spirv(compute(threads(64)))] From 80448638cfc278107775e3da4e31b42d0dd071da Mon Sep 17 00:00:00 2001 From: firestar99 Date: Tue, 23 Sep 2025 11:02:27 +0200 Subject: [PATCH 16/17] difftest: typo fixes --- tests/difftests/runner/src/testcase.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/difftests/runner/src/testcase.rs b/tests/difftests/runner/src/testcase.rs index 9715abe25b..fb5b4eb9e8 100644 --- a/tests/difftests/runner/src/testcase.rs +++ b/tests/difftests/runner/src/testcase.rs @@ -9,9 +9,9 @@ use tracing::debug; pub struct TestCase { /// The name of the testcase, as a rust mod path pub name: String, - /// the relative path from the base `difftest` dir + /// The relative path from the base `difftest` dir pub relative_path: PathBuf, - /// the absolute path + /// The absolute path pub absolute_path: PathBuf, /// All the test binaries of this single test case. pub test_binaries: Vec, From 938c5111857bef857edb4e3d044b99e3248a4d4d Mon Sep 17 00:00:00 2001 From: firestar99 Date: Tue, 23 Sep 2025 11:11:52 +0200 Subject: [PATCH 17/17] difftest: document nextest requirement --- default.nix | 1 + tests/difftests/README.md | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/default.nix b/default.nix index 3194b5f017..4b907d8dd0 100644 --- a/default.nix +++ b/default.nix @@ -12,6 +12,7 @@ in with pkgs; stdenv.mkDerivation rec { SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; nativeBuildInputs = [ rustup ]; + buildInputs = [ cargo-nextest ]; # Runtime dependencies (for the example runners). LD_LIBRARY_PATH = with xorg; lib.makeLibraryPath [ diff --git a/tests/difftests/README.md b/tests/difftests/README.md index 8edb58f144..8c71bf242a 100644 --- a/tests/difftests/README.md +++ b/tests/difftests/README.md @@ -160,6 +160,12 @@ config.write_metadata(&metadata)?; ## Running Tests +### Install cargo-nextest: + +```sh +cargo install cargo-nextest +``` + ### Run all difftests: ```sh