diff --git a/src/test/ui/json-multiple.nll.stderr b/src/test/ui/json-multiple.nll.stderr
deleted file mode 100644
index c2cb8bcde19..00000000000
--- a/src/test/ui/json-multiple.nll.stderr
+++ /dev/null
@@ -1 +0,0 @@
-{"artifact":"$TEST_BUILD_DIR/json-multiple.nll/libjson_multiple.rlib","emit":"link"}
diff --git a/src/test/ui/json-multiple.rs b/src/test/ui/json-multiple.rs
index fb126339dc2..4c37e20d94d 100644
--- a/src/test/ui/json-multiple.rs
+++ b/src/test/ui/json-multiple.rs
@@ -1,5 +1,6 @@
 // build-pass
 // ignore-pass (different metadata emitted in different modes)
 // compile-flags: --json=diagnostic-short --json artifacts --error-format=json
+// ignore-compare-mode-nll
 
 #![crate_type = "lib"]
diff --git a/src/test/ui/json-options.nll.stderr b/src/test/ui/json-options.nll.stderr
deleted file mode 100644
index f19a9cd92af..00000000000
--- a/src/test/ui/json-options.nll.stderr
+++ /dev/null
@@ -1 +0,0 @@
-{"artifact":"$TEST_BUILD_DIR/json-options.nll/libjson_options.rlib","emit":"link"}
diff --git a/src/test/ui/json-options.rs b/src/test/ui/json-options.rs
index 8b6ba131eb0..fea07cc9e3e 100644
--- a/src/test/ui/json-options.rs
+++ b/src/test/ui/json-options.rs
@@ -1,5 +1,6 @@
 // build-pass
 // ignore-pass (different metadata emitted in different modes)
 // compile-flags: --json=diagnostic-short,artifacts --error-format=json
+// ignore-compare-mode-nll
 
 #![crate_type = "lib"]
diff --git a/src/test/ui/rmeta/emit-artifact-notifications.nll.stderr b/src/test/ui/rmeta/emit-artifact-notifications.nll.stderr
deleted file mode 100644
index ed62f300284..00000000000
--- a/src/test/ui/rmeta/emit-artifact-notifications.nll.stderr
+++ /dev/null
@@ -1 +0,0 @@
-{"artifact":"$TEST_BUILD_DIR/rmeta/emit-artifact-notifications.nll/libemit_artifact_notifications.rmeta","emit":"metadata"}
diff --git a/src/test/ui/rmeta/emit-artifact-notifications.rs b/src/test/ui/rmeta/emit-artifact-notifications.rs
index 984a7fabb66..be38fb4c3a6 100644
--- a/src/test/ui/rmeta/emit-artifact-notifications.rs
+++ b/src/test/ui/rmeta/emit-artifact-notifications.rs
@@ -2,6 +2,7 @@
 // build-pass
 // ignore-pass
 // ^-- needed because `--pass check` does not emit the output needed.
+// ignore-compare-mode-nll
 
 // A very basic test for the emission of artifact notifications in JSON output.
 
diff --git a/src/test/ui/save-analysis/emit-notifications.nll.stderr b/src/test/ui/save-analysis/emit-notifications.nll.stderr
deleted file mode 100644
index 60734cedb50..00000000000
--- a/src/test/ui/save-analysis/emit-notifications.nll.stderr
+++ /dev/null
@@ -1,2 +0,0 @@
-{"artifact":"$TEST_BUILD_DIR/save-analysis/emit-notifications.nll/save-analysis/libemit_notifications.json","emit":"save-analysis"}
-{"artifact":"$TEST_BUILD_DIR/save-analysis/emit-notifications.nll/libemit_notifications.rlib","emit":"link"}
diff --git a/src/test/ui/save-analysis/emit-notifications.rs b/src/test/ui/save-analysis/emit-notifications.rs
index 9179944a620..8a696695ec0 100644
--- a/src/test/ui/save-analysis/emit-notifications.rs
+++ b/src/test/ui/save-analysis/emit-notifications.rs
@@ -3,5 +3,6 @@
 // compile-flags: --crate-type rlib --error-format=json
 // ignore-pass
 // ^-- needed because otherwise, the .stderr file changes with --pass check
+// ignore-compare-mode-nll
 
 pub fn foo() {}
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index a59a0584d5e..64b6df3567a 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -3112,7 +3112,7 @@ impl<'test> TestCx<'test> {
         let expected_fixed = self.load_expected_output(UI_FIXED);
 
         let modes_to_prune = vec![CompareMode::Nll];
-        self.prune_duplicate_outputs(&modes_to_prune);
+        self.check_and_prune_duplicate_outputs(&proc_res, &[], &modes_to_prune);
 
         let mut errors = self.load_compare_outputs(&proc_res, TestOutput::Compile, explicit);
         let rustfix_input = json::rustfix_diagnostics_only(&proc_res.stderr);
@@ -3730,28 +3730,54 @@ impl<'test> TestCx<'test> {
         if self.config.bless { 0 } else { 1 }
     }
 
-    fn prune_duplicate_output(&self, mode: CompareMode, kind: &str, canon_content: &str) {
-        let examined_path = expected_output_path(&self.testpaths, self.revision, &Some(mode), kind);
+    fn check_and_prune_duplicate_outputs(
+        &self,
+        proc_res: &ProcRes,
+        modes: &[CompareMode],
+        require_same_modes: &[CompareMode],
+    ) {
+        for kind in UI_EXTENSIONS {
+            let canon_comparison_path =
+                expected_output_path(&self.testpaths, self.revision, &None, kind);
 
-        let examined_content =
-            self.load_expected_output_from_path(&examined_path).unwrap_or_else(|_| String::new());
+            let canon = match self.load_expected_output_from_path(&canon_comparison_path) {
+                Ok(canon) => canon,
+                _ => continue,
+            };
+            let bless = self.config.bless;
+            let check_and_prune_duplicate_outputs = |mode: &CompareMode, require_same: bool| {
+                let examined_path =
+                    expected_output_path(&self.testpaths, self.revision, &Some(mode.clone()), kind);
 
-        if canon_content == examined_content {
-            self.delete_file(&examined_path);
-        }
-    }
+                // If there is no output, there is nothing to do
+                let examined_content = match self.load_expected_output_from_path(&examined_path) {
+                    Ok(content) => content,
+                    _ => return,
+                };
 
-    fn prune_duplicate_outputs(&self, modes: &[CompareMode]) {
-        if self.config.bless {
-            for kind in UI_EXTENSIONS {
-                let canon_comparison_path =
-                    expected_output_path(&self.testpaths, self.revision, &None, kind);
+                let is_duplicate = canon == examined_content;
 
-                if let Ok(canon) = self.load_expected_output_from_path(&canon_comparison_path) {
-                    for mode in modes {
-                        self.prune_duplicate_output(mode.clone(), kind, &canon);
+                match (bless, require_same, is_duplicate) {
+                    // If we're blessing and the output is the same, then delete the file.
+                    (true, _, true) => {
+                        self.delete_file(&examined_path);
                     }
+                    // If we want them to be the same, but they are different, then error.
+                    // We do this wether we bless or not
+                    (_, true, false) => {
+                        self.fatal_proc_rec(
+                            &format!("`{}` should not have different output from base test!", kind),
+                            proc_res,
+                        );
+                    }
+                    _ => {}
                 }
+            };
+            for mode in modes {
+                check_and_prune_duplicate_outputs(mode, false);
+            }
+            for mode in require_same_modes {
+                check_and_prune_duplicate_outputs(mode, true);
             }
         }
     }