mathieu/granite-rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Auto merge of #110444 - ferrocene:comptest-lines, r=Mark-Simulacrum

Browse source 
Add compare-output-lines-by-subset flag to compiletest

For [ferrocene](github.com/ferrocene/) we have some compiletests that check the output of the cli arguments to the compiler, including printing things like the target list (`--print target-list`). Unfortunately those tend to change quite often so when we sync we end up with some outputs we have to re-bless constantly, even though the exact output doesn't really matter.

We added a new compiletest flag to aid writing these kinds of tests: `compare-output-lines-by-subset`. It checks whether the lines of the expected output are a subset (or equal) to the lines of the actual output. If the expected output is empty it will fail unless the actual output is also empty. We opened this PR hoping the flag might be helpful for other tests in the future (especially if CLI-related tests are added in the future in the rust-lang/rust repo itself).
This commit is contained in:
bors 2023-04-20 15:15:17 +00:00
commit a57fa08f48
2 changed files with 68 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/tools/compiletest/src/header.rs
View file

@ -102,6 +102,9 @@ pub struct TestProps {
pub dont_check_compiler_stdout: bool,
// For UI tests, allows compiler to generate arbitrary output to stderr
pub dont_check_compiler_stderr: bool,
// When checking the output of stdout or stderr check
// that the lines of expected output are a subset of the actual output.
pub compare_output_lines_by_subset: bool,
// Don't force a --crate-type=dylib flag on the command line
//
// Set this for example if you have an auxiliary test file that contains
@ -209,6 +212,7 @@ mod directives {
pub const KNOWN_BUG: &'static str = "known-bug";
pub const MIR_UNIT_TEST: &'static str = "unit-test";
pub const REMAP_SRC_BASE: &'static str = "remap-src-base";
pub const COMPARE_OUTPUT_LINES_BY_SUBSET: &'static str = "compare-output-lines-by-subset";
// This isn't a real directive, just one that is probably mistyped often
pub const INCORRECT_COMPILER_FLAGS: &'static str = "compiler-flags";
}
@ -233,6 +237,7 @@ impl TestProps {
check_run_results: false,
dont_check_compiler_stdout: false,
dont_check_compiler_stderr: false,
compare_output_lines_by_subset: false,
no_prefer_dynamic: false,
pretty_expanded: false,
pretty_mode: "normal".to_string(),
@ -467,6 +472,11 @@ impl TestProps {
s.trim().to_string()
});
config.set_name_directive(ln, REMAP_SRC_BASE, &mut self.remap_src_base);
config.set_name_directive(
ln,
COMPARE_OUTPUT_LINES_BY_SUBSET,
&mut self.compare_output_lines_by_subset,
);
});
}

66
src/tools/compiletest/src/runtest.rs
View file

@ -3244,17 +3244,35 @@ impl<'test> TestCx<'test> {
match output_kind {
TestOutput::Compile => {
if !self.props.dont_check_compiler_stdout {
errors +=
self.compare_output(stdout_kind, &normalized_stdout, &expected_stdout);
errors += self.compare_output(
stdout_kind,
&normalized_stdout,
&expected_stdout,
self.props.compare_output_lines_by_subset,
);
}
if !self.props.dont_check_compiler_stderr {
errors +=
self.compare_output(stderr_kind, &normalized_stderr, &expected_stderr);
errors += self.compare_output(
stderr_kind,
&normalized_stderr,
&expected_stderr,
self.props.compare_output_lines_by_subset,
);
}
}
TestOutput::Run => {
errors += self.compare_output(stdout_kind, &normalized_stdout, &expected_stdout);
errors += self.compare_output(stderr_kind, &normalized_stderr, &expected_stderr);
errors += self.compare_output(
stdout_kind,
&normalized_stdout,
&expected_stdout,
self.props.compare_output_lines_by_subset,
);
errors += self.compare_output(
stderr_kind,
&normalized_stderr,
&expected_stderr,
self.props.compare_output_lines_by_subset,
);
}
}
errors
@ -3338,7 +3356,12 @@ impl<'test> TestCx<'test> {
)
});
errors += self.compare_output("fixed", &fixed_code, &expected_fixed);
errors += self.compare_output(
"fixed",
&fixed_code,
&expected_fixed,
self.props.compare_output_lines_by_subset,
);
} else if !expected_fixed.is_empty() {
panic!(
"the `// run-rustfix` directive wasn't found but a `*.fixed` \
@ -3796,11 +3819,38 @@ impl<'test> TestCx<'test> {
}
}
fn compare_output(&self, kind: &str, actual: &str, expected: &str) -> usize {
fn compare_output(
&self,
kind: &str,
actual: &str,
expected: &str,
compare_output_by_lines: bool,
) -> usize {
if actual == expected {
return 0;
}
let tmp;
let (expected, actual): (&str, &str) = if compare_output_by_lines {
let actual_lines: HashSet<_> = actual.lines().collect();
let expected_lines: Vec<_> = expected.lines().collect();
let mut used = expected_lines.clone();
used.retain(|line| actual_lines.contains(line));
// check if `expected` contains a subset of the lines of `actual`
if used.len() == expected_lines.len() && (expected.is_empty() == actual.is_empty()) {
return 0;
}
if expected_lines.is_empty() {
// if we have no lines to check, force a full overwite
("", actual)
} else {
tmp = (expected_lines.join("\n"), used.join("\n"));
(&tmp.0, &tmp.1)
}
} else {
(expected, actual)
};
if !self.config.bless {
if expected.is_empty() {
println!("normalized {}:\n{}\n", kind, actual);