mathieu/granite-rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Auto merge of #109371 - Zoxc:verify-hash-opt, r=cjgillot

Browse source 
Optimize `incremental_verify_ich`

This optimizes `incremental_verify_ich` by operating on `SerializedDepNodeIndex`, saving 2 hashmap lookups. The panic paths are also changed to get a `TyCtxt` reference using TLS.
This commit is contained in:
bors 2023-03-25 09:22:15 +00:00
commit e10eab5956
3 changed files with 66 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
compiler/rustc_query_system/src/dep_graph/graph.rs
View file

@ -535,16 +535,22 @@ impl<K: DepKind> DepGraph<K> {
// value to an existing node.
//
// For sanity, we still check that the loaded stable hash and the new one match.
if let Some(dep_node_index) = data.dep_node_index_of_opt(&node) {
let _current_fingerprint =
crate::query::incremental_verify_ich(cx, data, result, &node, hash_result);
if let Some(prev_index) = data.previous.node_to_index_opt(&node) {
let dep_node_index = data.current.prev_index_to_index.lock()[prev_index];
if let Some(dep_node_index) = dep_node_index {
crate::query::incremental_verify_ich(cx, data, result, prev_index, hash_result);
#[cfg(debug_assertions)]
if hash_result.is_some() {
data.current.record_edge(dep_node_index, node, _current_fingerprint);
#[cfg(debug_assertions)]
if hash_result.is_some() {
data.current.record_edge(
dep_node_index,
node,
data.prev_fingerprint_of(prev_index),
);
}
return dep_node_index;
}
return dep_node_index;
}
let mut edges = SmallVec::new();
@ -626,13 +632,19 @@ impl<K: DepKind> DepGraphData<K> {
/// Returns true if the given node has been marked as green during the
/// current compilation session. Used in various assertions
pub fn is_green(&self, dep_node: &DepNode<K>) -> bool {
self.node_color(dep_node).map_or(false, |c| c.is_green())
#[inline]
pub fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool {
self.colors.get(prev_index).map_or(false, |c| c.is_green())
}
#[inline]
pub fn prev_fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
self.previous.fingerprint_of(dep_node)
pub fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
self.previous.fingerprint_by_index(prev_index)
}
#[inline]
pub fn prev_node_of(&self, prev_index: SerializedDepNodeIndex) -> DepNode<K> {
self.previous.index_to_node(prev_index)
}
pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode<K>) {
@ -643,7 +655,7 @@ impl<K: DepKind> DepGraphData<K> {
impl<K: DepKind> DepGraph<K> {
#[inline]
pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
self.data.as_ref().and_then(|data| data.dep_node_index_of_opt(dep_node)).is_some()
self.data.as_ref().map_or(false, |data| data.dep_node_exists(dep_node))
}
/// Checks whether a previous work product exists for `v` and, if

5
compiler/rustc_query_system/src/dep_graph/serialized.rs
View file

@ -79,11 +79,6 @@ impl<K: DepKind> SerializedDepGraph<K> {
self.index.get(dep_node).cloned()
}
#[inline]
pub fn fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
self.index.get(dep_node).map(|&node_index| self.fingerprints[node_index])
}
#[inline]
pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint {
self.fingerprints[dep_node_index]

108
compiler/rustc_query_system/src/query/plumbing.rs
View file

@ -7,6 +7,7 @@ use crate::dep_graph::{DepGraphData, HasDepContext};
use crate::ich::StableHashingContext;
use crate::query::caches::QueryCache;
use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
use crate::query::SerializedDepNodeIndex;
use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame};
use crate::values::Value;
use crate::HandleCycleError;
@ -19,7 +20,6 @@ use rustc_data_structures::sharded::Sharded;
use rustc_data_structures::stack::ensure_sufficient_stack;
use rustc_data_structures::sync::{Lock, LockGuard};
use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError};
use rustc_session::Session;
use rustc_span::{Span, DUMMY_SP};
use std::cell::Cell;
use std::collections::hash_map::Entry;
@ -537,7 +537,7 @@ where
let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, &dep_node)?;
debug_assert!(dep_graph_data.is_green(dep_node));
debug_assert!(dep_graph_data.is_index_green(prev_dep_node_index));
// First we try to load the result from the on-disk cache.
// Some things are never cached on disk.
@ -561,8 +561,7 @@ where
dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
}
let prev_fingerprint =
dep_graph_data.prev_fingerprint_of(dep_node).unwrap_or(Fingerprint::ZERO);
let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index);
// If `-Zincremental-verify-ich` is specified, re-hash results from
// the cache and make sure that they have the expected fingerprint.
//
@ -578,7 +577,7 @@ where
*qcx.dep_context(),
dep_graph_data,
&result,
dep_node,
prev_dep_node_index,
query.hash_result(),
);
}
@ -623,7 +622,7 @@ where
*qcx.dep_context(),
dep_graph_data,
&result,
dep_node,
prev_dep_node_index,
query.hash_result(),
);
@ -636,77 +635,50 @@ pub(crate) fn incremental_verify_ich<Tcx, V: Debug>(
tcx: Tcx,
dep_graph_data: &DepGraphData<Tcx::DepKind>,
result: &V,
dep_node: &DepNode<Tcx::DepKind>,
prev_index: SerializedDepNodeIndex,
hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>,
) -> Fingerprint
where
) where
Tcx: DepContext,
{
assert!(
dep_graph_data.is_green(dep_node),
"fingerprint for green query instance not loaded from cache: {dep_node:?}",
);
if !dep_graph_data.is_index_green(prev_index) {
incremental_verify_ich_not_green(tcx, prev_index)
}
let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| {
tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result))
});
let old_hash = dep_graph_data.prev_fingerprint_of(dep_node);
let old_hash = dep_graph_data.prev_fingerprint_of(prev_index);
if Some(new_hash) != old_hash {
incremental_verify_ich_failed(
tcx.sess(),
DebugArg::from(&dep_node),
DebugArg::from(&result),
);
}
new_hash
}
// This DebugArg business is largely a mirror of std::fmt::ArgumentV1, which is
// currently not exposed publicly.
//
// The PR which added this attempted to use `&dyn Debug` instead, but that
// showed statistically significant worse compiler performance. It's not
// actually clear what the cause there was -- the code should be cold. If this
// can be replaced with `&dyn Debug` with on perf impact, then it probably
// should be.
extern "C" {
type Opaque;
}
struct DebugArg<'a> {
value: &'a Opaque,
fmt: fn(&Opaque, &mut std::fmt::Formatter<'_>) -> std::fmt::Result,
}
impl<'a, T> From<&'a T> for DebugArg<'a>
where
T: std::fmt::Debug,
{
fn from(value: &'a T) -> DebugArg<'a> {
DebugArg {
value: unsafe { std::mem::transmute(value) },
fmt: unsafe {
std::mem::transmute(<T as std::fmt::Debug>::fmt as fn(_, _) -> std::fmt::Result)
},
}
if new_hash != old_hash {
incremental_verify_ich_failed(tcx, prev_index, result);
}
}
impl std::fmt::Debug for DebugArg<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
(self.fmt)(self.value, f)
}
}
// Note that this is marked #[cold] and intentionally takes the equivalent of
// `dyn Debug` for its arguments, as we want to avoid generating a bunch of
// different implementations for LLVM to chew on (and filling up the final
// binary, too).
#[cold]
fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result: DebugArg<'_>) {
#[inline(never)]
fn incremental_verify_ich_not_green<Tcx>(tcx: Tcx, prev_index: SerializedDepNodeIndex)
where
Tcx: DepContext,
{
panic!(
"fingerprint for green query instance not loaded from cache: {:?}",
tcx.dep_graph().data().unwrap().prev_node_of(prev_index)
)
}
// Note that this is marked #[cold] and intentionally takes `dyn Debug` for `result`,
// as we want to avoid generating a bunch of different implementations for LLVM to
// chew on (and filling up the final binary, too).
#[cold]
#[inline(never)]
fn incremental_verify_ich_failed<Tcx>(
tcx: Tcx,
prev_index: SerializedDepNodeIndex,
result: &dyn Debug,
) where
Tcx: DepContext,
{
// When we emit an error message and panic, we try to debug-print the `DepNode`
// and query result. Unfortunately, this can cause us to run additional queries,
// which may result in another fingerprint mismatch while we're in the middle
@ -720,15 +692,17 @@ fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result:
let old_in_panic = INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.replace(true));
if old_in_panic {
sess.emit_err(crate::error::Reentrant);
tcx.sess().emit_err(crate::error::Reentrant);
} else {
let run_cmd = if let Some(crate_name) = &sess.opts.crate_name {
let run_cmd = if let Some(crate_name) = &tcx.sess().opts.crate_name {
format!("`cargo clean -p {crate_name}` or `cargo clean`")
} else {
"`cargo clean`".to_string()
};
sess.emit_err(crate::error::IncrementCompilation {
let dep_node = tcx.dep_graph().data().unwrap().prev_node_of(prev_index);
let dep_node = tcx.sess().emit_err(crate::error::IncrementCompilation {
run_cmd,
dep_node: format!("{dep_node:?}"),
});