mathieu/os-rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement JumpThreading pass.

Browse source
This commit is contained in:
Camille GILLOT 2023-01-16 22:12:36 +00:00
parent 3cb0c2e385
commit 751a079413
27 changed files with 2375 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock
View file

@ -4242,6 +4242,7 @@ dependencies = [
"coverage_test_macros",
"either",
"itertools",
"rustc_arena",
"rustc_ast",
"rustc_attr",
"rustc_const_eval",

9
compiler/rustc_middle/src/mir/terminator.rs
View file

@ -28,6 +28,15 @@ impl SwitchTargets {
Self { values: smallvec![value], targets: smallvec![then, else_] }
}
/// Inverse of `SwitchTargets::static_if`.
pub fn as_static_if(&self) -> Option<(u128, BasicBlock, BasicBlock)> {
if let &[value] = &self.values[..] && let &[then, else_] = &self.targets[..] {
Some((value, then, else_))
} else {
None
}
}
/// Returns the fallback target that is jumped to when none of the values match the operand.
pub fn otherwise(&self) -> BasicBlock {
*self.targets.last().unwrap()

107
compiler/rustc_mir_dataflow/src/value_analysis.rs
View file

@ -463,7 +463,19 @@ impl<V: Clone> Clone for State<V> {
}
}
impl<V: Clone + HasTop + HasBottom> State<V> {
impl<V: Clone> State<V> {
pub fn new(init: V, map: &Map) -> State<V> {
let values = IndexVec::from_elem_n(init, map.value_count);
State(StateData::Reachable(values))
}
pub fn all(&self, f: impl Fn(&V) -> bool) -> bool {
match self.0 {
StateData::Unreachable => true,
StateData::Reachable(ref values) => values.iter().all(f),
}
}
pub fn is_reachable(&self) -> bool {
matches!(&self.0, StateData::Reachable(_))
}
@ -472,7 +484,10 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
self.0 = StateData::Unreachable;
}
pub fn flood_all(&mut self) {
pub fn flood_all(&mut self)
where
V: HasTop,
{
self.flood_all_with(V::TOP)
}
@ -482,27 +497,40 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
}
pub fn flood_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) {
let StateData::Reachable(values) = &mut self.0 else { return };
map.for_each_aliasing_place(place, None, &mut |vi| {
values[vi] = value.clone();
});
self.flood_with_extra(place, None, map, value)
}
pub fn flood(&mut self, place: PlaceRef<'_>, map: &Map) {
pub fn flood(&mut self, place: PlaceRef<'_>, map: &Map)
where
V: HasTop,
{
self.flood_with(place, map, V::TOP)
}
pub fn flood_discr_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) {
let StateData::Reachable(values) = &mut self.0 else { return };
map.for_each_aliasing_place(place, Some(TrackElem::Discriminant), &mut |vi| {
values[vi] = value.clone();
});
self.flood_with_extra(place, Some(TrackElem::Discriminant), map, value)
}
pub fn flood_discr(&mut self, place: PlaceRef<'_>, map: &Map) {
pub fn flood_discr(&mut self, place: PlaceRef<'_>, map: &Map)
where
V: HasTop,
{
self.flood_discr_with(place, map, V::TOP)
}
pub fn flood_with_extra(
&mut self,
place: PlaceRef<'_>,
tail_elem: Option<TrackElem>,
map: &Map,
value: V,
) {
let StateData::Reachable(values) = &mut self.0 else { return };
map.for_each_aliasing_place(place, tail_elem, &mut |vi| {
values[vi] = value.clone();
});
}
/// Low-level method that assigns to a place.
/// This does nothing if the place is not tracked.
///
@ -553,7 +581,10 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
}
/// Helper method to interpret `target = result`.
pub fn assign(&mut self, target: PlaceRef<'_>, result: ValueOrPlace<V>, map: &Map) {
pub fn assign(&mut self, target: PlaceRef<'_>, result: ValueOrPlace<V>, map: &Map)
where
V: HasTop,
{
self.flood(target, map);
if let Some(target) = map.find(target) {
self.insert_idx(target, result, map);
@ -561,7 +592,10 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
}
/// Helper method for assignments to a discriminant.
pub fn assign_discr(&mut self, target: PlaceRef<'_>, result: ValueOrPlace<V>, map: &Map) {
pub fn assign_discr(&mut self, target: PlaceRef<'_>, result: ValueOrPlace<V>, map: &Map)
where
V: HasTop,
{
self.flood_discr(target, map);
if let Some(target) = map.find_discr(target) {
self.insert_idx(target, result, map);
@ -569,12 +603,43 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
}
/// Retrieve the value stored for a place, or ⊤ if it is not tracked.
pub fn get(&self, place: PlaceRef<'_>, map: &Map) -> V {
pub fn try_get(&self, place: PlaceRef<'_>, map: &Map) -> Option<V> {
let place = map.find(place)?;
self.try_get_idx(place, map)
}
/// Retrieve the value stored for a place, or ⊤ if it is not tracked.
pub fn try_get_discr(&self, place: PlaceRef<'_>, map: &Map) -> Option<V> {
let place = map.find_discr(place)?;
self.try_get_idx(place, map)
}
/// Retrieve the value stored for a place index, or ⊤ if it is not tracked.
pub fn try_get_idx(&self, place: PlaceIndex, map: &Map) -> Option<V> {
match &self.0 {
StateData::Reachable(values) => {
map.places[place].value_index.map(|v| values[v].clone())
}
StateData::Unreachable => {
// Because this is unreachable, we can return any value we want.
None
}
}
}
/// Retrieve the value stored for a place, or ⊤ if it is not tracked.
pub fn get(&self, place: PlaceRef<'_>, map: &Map) -> V
where
V: HasBottom + HasTop,
{
map.find(place).map(|place| self.get_idx(place, map)).unwrap_or(V::TOP)
}
/// Retrieve the value stored for a place, or ⊤ if it is not tracked.
pub fn get_discr(&self, place: PlaceRef<'_>, map: &Map) -> V {
pub fn get_discr(&self, place: PlaceRef<'_>, map: &Map) -> V
where
V: HasBottom + HasTop,
{
match map.find_discr(place) {
Some(place) => self.get_idx(place, map),
None => V::TOP,
@ -582,7 +647,10 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
}
/// Retrieve the value stored for a place, or ⊤ if it is not tracked.
pub fn get_len(&self, place: PlaceRef<'_>, map: &Map) -> V {
pub fn get_len(&self, place: PlaceRef<'_>, map: &Map) -> V
where
V: HasBottom + HasTop,
{
match map.find_len(place) {
Some(place) => self.get_idx(place, map),
None => V::TOP,
@ -590,7 +658,10 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
}
/// Retrieve the value stored for a place index, or ⊤ if it is not tracked.
pub fn get_idx(&self, place: PlaceIndex, map: &Map) -> V {
pub fn get_idx(&self, place: PlaceIndex, map: &Map) -> V
where
V: HasBottom + HasTop,
{
match &self.0 {
StateData::Reachable(values) => {
map.places[place].value_index.map(|v| values[v].clone()).unwrap_or(V::TOP)

1
compiler/rustc_mir_transform/Cargo.toml
View file

@ -11,6 +11,7 @@ smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
tracing = "0.1"
either = "1"
rustc_ast = { path = "../rustc_ast" }
rustc_arena = { path = "../rustc_arena" }
rustc_attr = { path = "../rustc_attr" }
rustc_data_structures = { path = "../rustc_data_structures" }
rustc_errors = { path = "../rustc_errors" }

638
compiler/rustc_mir_transform/src/jump_threading.rs Normal file
View file

@ -0,0 +1,638 @@
//! A jump threading optimization.
//!
//! This optimization seeks to replace join-then-switch control flow patterns by straight jumps
//! X = 0 X = 0
//! ------------\ /-------- ------------
//! X = 1 X----X SwitchInt(X) => X = 1
//! ------------/ \-------- ------------
//!
//!
//! We proceed by walking the cfg backwards starting from each `SwitchInt` terminator,
//! looking for assignments that will turn the `SwitchInt` into a simple `Goto`.
//!
//! The algorithm maintains a set of replacement conditions:
//! - `conditions[place]` contains `Condition { value, polarity: true, target }`
//! if assigning `value` to `place` turns the `SwitchInt` into `Goto { target }`.
//! - `conditions[place]` contains `Condition { value, polarity: false, target }`
//! if assigning anything different from `value` to `place` turns the `SwitchInt`
//! into `Goto { target }`.
//!
//! We then walk the CFG backwards transforming the set of conditions.
//! When we find a fulfilling assignment, we record a `ThreadingOpportunity`.
//! All `ThreadingOpportunity`s are applied to the body, by duplicating blocks if required.
//!
//! The optimization search can be very heavy, as it performs a DFS on MIR starting from
//! each `SwitchInt` terminator. To manage the complexity, we:
//! - bound the maximum depth by a constant `MAX_BACKTRACK`;
//! - we only traverse `Goto` terminators.
//!
//! Likewise, applying the optimisation can create a lot of new MIR, so we bound the instruction
//! cost by `MAX_COST`.
use rustc_arena::DroplessArena;
use rustc_data_structures::fx::FxHashSet;
use rustc_index::IndexVec;
use rustc_middle::mir::visit::Visitor;
use rustc_middle::mir::*;
use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt};
use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, State, TrackElem};
use crate::cost_checker::CostChecker;
use crate::MirPass;
pub struct JumpThreading;
const MAX_BACKTRACK: usize = 5;
const MAX_COST: usize = 100;
const MAX_PLACES: usize = 100;
impl<'tcx> MirPass<'tcx> for JumpThreading {
fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
sess.mir_opt_level() >= 4
}
#[instrument(skip_all level = "debug")]
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
let def_id = body.source.def_id();
debug!(?def_id);
let param_env = tcx.param_env_reveal_all_normalized(def_id);
let map = Map::new(tcx, body, Some(MAX_PLACES));
let arena = DroplessArena::default();
let mut finder =
TOFinder { tcx, param_env, body, arena: &arena, map: &map, opportunities: Vec::new() };
for (bb, bbdata) in body.basic_blocks.iter_enumerated() {
debug!(?bb, term = ?bbdata.terminator());
if bbdata.is_cleanup {
continue;
}
let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { continue };
let Some(discr) = discr.place() else { continue };
debug!(?discr, ?bb);
let discr_ty = discr.ty(body, tcx).ty;
let Ok(discr_layout) = tcx.layout_of(param_env.and(discr_ty)) else { continue };
let Some(discr) = finder.map.find(discr.as_ref()) else { continue };
debug!(?discr);
let cost = CostChecker::new(tcx, param_env, None, body);
let mut state = State::new(ConditionSet::default(), &finder.map);
let conds = if let Some((value, then, else_)) = targets.as_static_if() {
let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else {
continue;
};
arena.alloc_from_iter([
Condition { value, polarity: true, target: then },
Condition { value, polarity: false, target: else_ },
])
} else {
arena.alloc_from_iter(targets.iter().filter_map(|(value, target)| {
let value = ScalarInt::try_from_uint(value, discr_layout.size)?;
Some(Condition { value, polarity: true, target })
}))
};
let conds = ConditionSet(conds);
state.insert_value_idx(discr, conds, &finder.map);
finder.find_opportunity(bb, state, cost, 0);
}
let opportunities = finder.opportunities;
debug!(?opportunities);
if opportunities.is_empty() {
return;
}
OpportunitySet::new(body, opportunities).apply(body);
}
}
#[derive(Debug)]
struct ThreadingOpportunity {
/// The list of `BasicBlock`s from the one that found the opportunity to the `SwitchInt`.
chain: Vec<BasicBlock>,
/// The `SwitchInt` will be replaced by `Goto { target }`.
target: BasicBlock,
}
struct TOFinder<'tcx, 'a> {
tcx: TyCtxt<'tcx>,
param_env: ty::ParamEnv<'tcx>,
body: &'a Body<'tcx>,
map: &'a Map,
/// We use an arena to avoid cloning the slices when cloning `state`.
arena: &'a DroplessArena,
opportunities: Vec<ThreadingOpportunity>,
}
#[derive(Copy, Clone, Debug)]
struct Condition {
value: ScalarInt,
/// `true` means `==`, `false` means `!=`
polarity: bool,
target: BasicBlock,
}
impl Condition {
fn matches(&self, value: ScalarInt) -> bool {
(self.value == value) == self.polarity
}
fn inv(mut self) -> Self {
self.polarity = !self.polarity;
self
}
}
#[derive(Copy, Clone, Debug, Default)]
struct ConditionSet<'a>(&'a [Condition]);
impl<'a> ConditionSet<'a> {
fn iter(self) -> impl Iterator<Item = Condition> + 'a {
self.0.iter().copied()
}
fn iter_matches(self, value: ScalarInt) -> impl Iterator<Item = Condition> + 'a {
self.iter().filter(move |c| c.matches(value))
}
fn map(self, arena: &'a DroplessArena, f: impl Fn(Condition) -> Condition) -> ConditionSet<'a> {
ConditionSet(arena.alloc_from_iter(self.iter().map(f)))
}
}
impl<'tcx, 'a> TOFinder<'tcx, 'a> {
fn is_empty(&self, state: &State<ConditionSet<'a>>) -> bool {
state.all(|cs| cs.0.is_empty())
}
/// Recursion entry point to find threading opportunities.
#[instrument(level = "trace", skip(self, cost), ret)]
fn find_opportunity(
&mut self,
bb: BasicBlock,
mut state: State<ConditionSet<'a>>,
mut cost: CostChecker<'_, 'tcx>,
depth: usize,
) {
debug!(cost = ?cost.cost());
for (statement_index, stmt) in
self.body.basic_blocks[bb].statements.iter().enumerate().rev()
{
if self.is_empty(&state) {
return;
}
cost.visit_statement(stmt, Location { block: bb, statement_index });
if cost.cost() > MAX_COST {
return;
}
// Attempt to turn the `current_condition` on `lhs` into a condition on another place.
self.process_statement(bb, stmt, &mut state);
// When a statement mutates a place, assignments to that place that happen
// above the mutation cannot fulfill a condition.
// _1 = 5 // Whatever happens here, it won't change the result of a `SwitchInt`.
// _1 = 6
if let Some((lhs, tail)) = self.mutated_statement(stmt) {
state.flood_with_extra(lhs.as_ref(), tail, self.map, ConditionSet::default());
}
}
if self.is_empty(&state) || depth >= MAX_BACKTRACK {
return;
}
let last_non_rec = self.opportunities.len();
let predecessors = &self.body.basic_blocks.predecessors()[bb];
if let &[pred] = &predecessors[..] && bb != START_BLOCK {
match &self.body.basic_blocks[pred].terminator().kind {
TerminatorKind::Goto { .. } => self.find_opportunity(pred, state, cost, depth),
TerminatorKind::SwitchInt { discr, targets } => {
self.process_switch_int(state, discr, targets, bb);
}
_ => {}
}
} else {
for &pred in predecessors {
if matches!(
self.body.basic_blocks[pred].terminator().kind,
TerminatorKind::Goto { .. }
) {
self.find_opportunity(pred, state.clone(), cost.clone(), depth + 1);
}
}
}
let new_tos = &mut self.opportunities[last_non_rec..];
debug!(?new_tos);
// Try to deduplicate threading opportunities.
if new_tos.len() > 1
&& new_tos.len() == predecessors.len()
&& predecessors
.iter()
.zip(new_tos.iter())
.all(|(&pred, to)| to.chain == &[pred] && to.target == new_tos[0].target)
{
// All predecessors have a threading opportunity, and they all point to the same block.
debug!(?new_tos, "dedup");
let first = &mut new_tos[0];
*first = ThreadingOpportunity { chain: vec![bb], target: first.target };
self.opportunities.truncate(last_non_rec + 1);
return;
}
for op in self.opportunities[last_non_rec..].iter_mut() {
op.chain.push(bb);
}
}
/// Extract the mutated place from a statement.
#[instrument(level = "trace", skip(self), ret)]
fn mutated_statement(
&self,
stmt: &Statement<'tcx>,
) -> Option<(Place<'tcx>, Option<TrackElem>)> {
match stmt.kind {
StatementKind::Assign(box (place, _))
| StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(
Operand::Copy(place) | Operand::Move(place),
))
| StatementKind::Deinit(box place) => Some((place, None)),
StatementKind::SetDiscriminant { box place, variant_index: _ } => {
Some((place, Some(TrackElem::Discriminant)))
}
StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => {
Some((Place::from(local), None))
}
StatementKind::Retag(..)
| StatementKind::Intrinsic(..)
| StatementKind::AscribeUserType(..)
| StatementKind::Coverage(..)
| StatementKind::FakeRead(..)
| StatementKind::ConstEvalCounter
| StatementKind::PlaceMention(..)
| StatementKind::Nop => None,
}
}
#[instrument(level = "trace", skip(self))]
fn process_operand(
&mut self,
bb: BasicBlock,
lhs: PlaceIndex,
rhs: &Operand<'tcx>,
state: &mut State<ConditionSet<'a>>,
) -> Option<!> {
let register_opportunity = |c: Condition| {
debug!(?bb, ?c.target, "register");
self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target })
};
match rhs {
// If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`.
Operand::Constant(constant) => {
let conditions = state.try_get_idx(lhs, self.map)?;
let constant =
constant.const_.normalize(self.tcx, self.param_env).try_to_scalar_int()?;
conditions.iter_matches(constant).for_each(register_opportunity);
}
// Transfer the conditions on the copied rhs.
Operand::Move(rhs) | Operand::Copy(rhs) => {
let rhs = self.map.find(rhs.as_ref())?;
state.insert_place_idx(rhs, lhs, self.map);
}
}
None
}
#[instrument(level = "trace", skip(self))]
fn process_statement(
&mut self,
bb: BasicBlock,
stmt: &Statement<'tcx>,
state: &mut State<ConditionSet<'a>>,
) -> Option<!> {
let register_opportunity = |c: Condition| {
debug!(?bb, ?c.target, "register");
self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target })
};
// Below, `lhs` is the return value of `mutated_statement`,
// the place to which `conditions` apply.
let discriminant_for_variant = |enum_ty: Ty<'tcx>, variant_index| {
let discr = enum_ty.discriminant_for_variant(self.tcx, variant_index)?;
let discr_layout = self.tcx.layout_of(self.param_env.and(discr.ty)).ok()?;
let scalar = ScalarInt::try_from_uint(discr.val, discr_layout.size)?;
Some(Operand::const_from_scalar(
self.tcx,
discr.ty,
scalar.into(),
rustc_span::DUMMY_SP,
))
};
match &stmt.kind {
// If we expect `discriminant(place) ?= A`,
// we have an opportunity if `variant_index ?= A`.
StatementKind::SetDiscriminant { box place, variant_index } => {
let discr_target = self.map.find_discr(place.as_ref())?;
let enum_ty = place.ty(self.body, self.tcx).ty;
let discr = discriminant_for_variant(enum_ty, *variant_index)?;
self.process_operand(bb, discr_target, &discr, state)?;
}
// If we expect `lhs ?= true`, we have an opportunity if we assume `lhs == true`.
StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(
Operand::Copy(place) | Operand::Move(place),
)) => {
let conditions = state.try_get(place.as_ref(), self.map)?;
conditions.iter_matches(ScalarInt::TRUE).for_each(register_opportunity);
}
StatementKind::Assign(box (lhs_place, rhs)) => {
if let Some(lhs) = self.map.find(lhs_place.as_ref()) {
match rhs {
Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state)?,
// Transfer the conditions on the copy rhs.
Rvalue::CopyForDeref(rhs) => {
self.process_operand(bb, lhs, &Operand::Copy(*rhs), state)?
}
Rvalue::Discriminant(rhs) => {
let rhs = self.map.find_discr(rhs.as_ref())?;
state.insert_place_idx(rhs, lhs, self.map);
}
// If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`.
Rvalue::Aggregate(box ref kind, ref operands) => {
let agg_ty = lhs_place.ty(self.body, self.tcx).ty;
let lhs = match kind {
// Do not support unions.
AggregateKind::Adt(.., Some(_)) => return None,
AggregateKind::Adt(_, variant_index, ..) if agg_ty.is_enum() => {
if let Some(discr_target) = self.map.apply(lhs, TrackElem::Discriminant)
&& let Some(discr_value) = discriminant_for_variant(agg_ty, *variant_index)
{
self.process_operand(bb, discr_target, &discr_value, state);
}
self.map.apply(lhs, TrackElem::Variant(*variant_index))?
}
_ => lhs,
};
for (field_index, operand) in operands.iter_enumerated() {
if let Some(field) =
self.map.apply(lhs, TrackElem::Field(field_index))
{
self.process_operand(bb, field, operand, state);
}
}
}
// Transfer the conditions on the copy rhs, after inversing polarity.
Rvalue::UnaryOp(UnOp::Not, Operand::Move(place) | Operand::Copy(place)) => {
let conditions = state.try_get_idx(lhs, self.map)?;
let place = self.map.find(place.as_ref())?;
let conds = conditions.map(self.arena, Condition::inv);
state.insert_value_idx(place, conds, self.map);
}
// We expect `lhs ?= A`. We found `lhs = Eq(rhs, B)`.
// Create a condition on `rhs ?= B`.
Rvalue::BinaryOp(
op,
box (
Operand::Move(place) | Operand::Copy(place),
Operand::Constant(value),
)
| box (
Operand::Constant(value),
Operand::Move(place) | Operand::Copy(place),
),
) => {
let conditions = state.try_get_idx(lhs, self.map)?;
let place = self.map.find(place.as_ref())?;
let equals = match op {
BinOp::Eq => ScalarInt::TRUE,
BinOp::Ne => ScalarInt::FALSE,
_ => return None,
};
let value = value
.const_
.normalize(self.tcx, self.param_env)
.try_to_scalar_int()?;
let conds = conditions.map(self.arena, |c| Condition {
value,
polarity: c.matches(equals),
..c
});
state.insert_value_idx(place, conds, self.map);
}
_ => {}
}
}
}
_ => {}
}
None
}
#[instrument(level = "trace", skip(self))]
fn process_switch_int(
&mut self,
state: State<ConditionSet<'a>>,
discr: &Operand<'tcx>,
targets: &SwitchTargets,
bb: BasicBlock,
) -> Option<!> {
debug_assert_ne!(bb, START_BLOCK);
debug_assert_eq!(self.body.basic_blocks.predecessors()[bb].len(), 1);
let discr = discr.place()?;
let discr_ty = discr.ty(self.body, self.tcx).ty;
let discr_layout = self.tcx.layout_of(self.param_env.and(discr_ty)).ok()?;
let conditions = state.try_get(discr.as_ref(), self.map)?;
if let Some((value, _)) = targets.iter().find(|&(_, target)| target == bb) {
let value = ScalarInt::try_from_uint(value, discr_layout.size)?;
debug_assert_eq!(targets.iter().filter(|&(_, target)| target == bb).count(), 1);
// We are inside `bb`. Since we have a single predecessor, we know we passed
// through the `SwitchInt` before arriving here. Therefore, we know that
// `discr == value`. If one condition can be fulfilled by `discr == value`,
// that's an opportunity.
for c in conditions.iter_matches(value) {
debug!(?bb, ?c.target, "register");
self.opportunities.push(ThreadingOpportunity { chain: vec![], target: c.target });
}
} else if bb == targets.otherwise() {
let (value, _, _) = targets.as_static_if()?;
let value = ScalarInt::try_from_uint(value, discr_layout.size)?;
// Likewise, we know that `discr != value`. That's a must weaker information,
// so we can only match the exact same condition.
for c in conditions.iter() {
if c.value == value && c.polarity == false {
debug!(?bb, ?c.target, "register");
self.opportunities
.push(ThreadingOpportunity { chain: vec![], target: c.target });
}
}
}
None
}
}
struct OpportunitySet {
opportunities: Vec<ThreadingOpportunity>,
/// For each bb, give the TOs in which it appears. The pair corresponds to the index
/// in `opportunities` and the index in `ThreadingOpportunity::chain`.
involving_tos: IndexVec<BasicBlock, Vec<(usize, usize)>>,
/// Cache the number of predecessors for each block, as we clear the basic block cache..
predecessors: IndexVec<BasicBlock, usize>,
}
impl OpportunitySet {
fn new(body: &Body<'_>, opportunities: Vec<ThreadingOpportunity>) -> OpportunitySet {
let mut involving_tos = IndexVec::from_elem(Vec::new(), &body.basic_blocks);
for (index, to) in opportunities.iter().enumerate() {
for (ibb, &bb) in to.chain.iter().enumerate() {
involving_tos[bb].push((index, ibb));
}
involving_tos[to.target].push((index, to.chain.len()));
}
let predecessors = predecessor_count(body);
OpportunitySet { opportunities, involving_tos, predecessors }
}
/// Apply the opportunities on the graph.
fn apply(&mut self, body: &mut Body<'_>) {
for i in 0..self.opportunities.len() {
self.apply_once(i, body);
}
}
#[instrument(level = "trace", skip(self, body))]
fn apply_once(&mut self, index: usize, body: &mut Body<'_>) {
debug!(?self.predecessors);
debug!(?self.involving_tos);
// Check that `predecessors` satisfies its invariant.
debug_assert_eq!(self.predecessors, predecessor_count(body));
// Remove the TO from the vector to allow modifying the other ones later.
let op = &mut self.opportunities[index];
debug!(?op);
let op_chain = std::mem::take(&mut op.chain);
let op_target = op.target;
debug_assert_eq!(op_chain.len(), op_chain.iter().collect::<FxHashSet<_>>().len());
let Some((current, chain)) = op_chain.split_first() else { return };
let basic_blocks = body.basic_blocks.as_mut();
// Invariant: we never change the meaning of the program.
let mut current = *current;
for &succ in chain {
debug!(?current, ?succ);
// `succ` must be a successor of `current`. If it is not, this means this TO is not
// satisfiable, so we bail out.
if basic_blocks[current].terminator().successors().find(|s| *s == succ).is_none() {
debug!("impossible");
return;
}
// Fast path: `succ` is only used once, so we can reuse it directly.
if self.predecessors[succ] == 1 {
debug!("single");
current = succ;
continue;
}
let new_succ = basic_blocks.push(basic_blocks[succ].clone());
debug!(?new_succ);
// Replace `succ` by `new_succ` where it appears.
let mut num_edges = 0;
for s in basic_blocks[current].terminator_mut().successors_mut() {
if *s == succ {
*s = new_succ;
num_edges += 1;
}
}
// Update predecessors with the new block.
let _new_succ = self.predecessors.push(num_edges);
debug_assert_eq!(new_succ, _new_succ);
self.predecessors[succ] -= num_edges;
self.update_predecessor_count(basic_blocks[new_succ].terminator(), Update::Incr);
// Replace the `current -> succ` edge by `current -> new_succ` in all the following
// TOs. This is necessary to avoid trying to thread through a non-existing edge. We
// use `involving_tos` here to avoid traversing the full set of TOs on each iteration.
let mut new_involved = Vec::new();
for &(to_index, in_to_index) in &self.involving_tos[current] {
// That TO has already been applied, do nothing.
if to_index <= index {
continue;
}
let other_to = &mut self.opportunities[to_index];
if other_to.chain.get(in_to_index) != Some(&current) {
continue;
}
let s = other_to.chain.get_mut(in_to_index + 1).unwrap_or(&mut other_to.target);
if *s == succ {
// `other_to` references the `current -> succ` edge, so replace `succ`.
*s = new_succ;
new_involved.push((to_index, in_to_index + 1));
}
}
// Following TOs new reference `new_succ`, so we will need to update them if we
// duplicate `new_succ` later.
let _new_succ = self.involving_tos.push(new_involved);
debug_assert_eq!(new_succ, _new_succ);
current = new_succ;
}
let current = &mut basic_blocks[current];
self.update_predecessor_count(current.terminator(), Update::Decr);
current.terminator_mut().kind = TerminatorKind::Goto { target: op_target };
self.predecessors[op_target] += 1;
}
fn update_predecessor_count(&mut self, terminator: &Terminator<'_>, incr: Update) {
match incr {
Update::Incr => {
for s in terminator.successors() {
self.predecessors[s] += 1;
}
}
Update::Decr => {
for s in terminator.successors() {
self.predecessors[s] -= 1;
}
}
}
}
}
fn predecessor_count(body: &Body<'_>) -> IndexVec<BasicBlock, usize> {
let mut predecessors: IndexVec<_, _> =
body.basic_blocks.predecessors().iter().map(|ps| ps.len()).collect();
predecessors[START_BLOCK] += 1; // Account for the implicit entry edge.
predecessors
}
enum Update {
Incr,
Decr,
}

2
compiler/rustc_mir_transform/src/lib.rs
View file

@ -82,6 +82,7 @@ mod function_item_references;
mod gvn;
pub mod inline;
mod instsimplify;
mod jump_threading;
mod large_enums;
mod lower_intrinsics;
mod lower_slice_len;
@ -572,6 +573,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
&dataflow_const_prop::DataflowConstProp,
&const_debuginfo::ConstDebugInfo,
&o1(simplify_branches::SimplifyConstCondition::AfterConstProp),
&jump_threading::JumpThreading,
&early_otherwise_branch::EarlyOtherwiseBranch,
&simplify_comparison_integral::SimplifyComparisonIntegral,
&dead_store_elimination::DeadStoreElimination,

57
tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff Normal file
View file

@ -0,0 +1,57 @@
- // MIR for `custom_discr` before JumpThreading
+ // MIR for `custom_discr` after JumpThreading
fn custom_discr(_1: bool) -> u8 {
debug x => _1;
let mut _0: u8;
let mut _2: CustomDiscr;
let mut _3: bool;
let mut _4: u8;
bb0: {
StorageLive(_2);
StorageLive(_3);
_3 = _1;
switchInt(move _3) -> [0: bb2, otherwise: bb1];
}
bb1: {
_2 = CustomDiscr::A;
- goto -> bb3;
+ goto -> bb7;
}
bb2: {
_2 = CustomDiscr::B;
goto -> bb3;
}
bb3: {
StorageDead(_3);
_4 = discriminant(_2);
- switchInt(move _4) -> [35: bb5, otherwise: bb4];
+ goto -> bb4;
}
bb4: {
_0 = const 13_u8;
goto -> bb6;
}
bb5: {
_0 = const 5_u8;
goto -> bb6;
}
bb6: {
StorageDead(_2);
return;
+ }
+
+ bb7: {
+ StorageDead(_3);
+ _4 = discriminant(_2);
+ goto -> bb5;
}
}

57
tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff Normal file
View file

@ -0,0 +1,57 @@
- // MIR for `custom_discr` before JumpThreading
+ // MIR for `custom_discr` after JumpThreading
fn custom_discr(_1: bool) -> u8 {
debug x => _1;
let mut _0: u8;
let mut _2: CustomDiscr;
let mut _3: bool;
let mut _4: u8;
bb0: {
StorageLive(_2);
StorageLive(_3);
_3 = _1;
switchInt(move _3) -> [0: bb2, otherwise: bb1];
}
bb1: {
_2 = CustomDiscr::A;
- goto -> bb3;
+ goto -> bb7;
}
bb2: {
_2 = CustomDiscr::B;
goto -> bb3;
}
bb3: {
StorageDead(_3);
_4 = discriminant(_2);
- switchInt(move _4) -> [35: bb5, otherwise: bb4];
+ goto -> bb4;
}
bb4: {
_0 = const 13_u8;
goto -> bb6;
}
bb5: {
_0 = const 5_u8;
goto -> bb6;
}
bb6: {
StorageDead(_2);
return;
+ }
+
+ bb7: {
+ StorageDead(_3);
+ _4 = discriminant(_2);
+ goto -> bb5;
}
}

87
tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff Normal file
View file

@ -0,0 +1,87 @@
- // MIR for `dfa` before JumpThreading
+ // MIR for `dfa` after JumpThreading
fn dfa() -> () {
let mut _0: ();
let mut _1: DFA;
let mut _2: !;
let mut _3: ();
let mut _4: isize;
let mut _5: DFA;
let mut _6: DFA;
let mut _7: DFA;
let mut _8: !;
scope 1 {
debug state => _1;
}
bb0: {
StorageLive(_1);
_1 = DFA::A;
StorageLive(_2);
- goto -> bb1;
+ goto -> bb7;
}
bb1: {
_4 = discriminant(_1);
- switchInt(move _4) -> [0: bb4, 1: bb5, 2: bb6, 3: bb2, otherwise: bb3];
+ goto -> bb2;
}
bb2: {
_0 = const ();
StorageDead(_2);
StorageDead(_1);
return;
}
bb3: {
unreachable;
}
bb4: {
StorageLive(_5);
_5 = DFA::B;
_1 = move _5;
_3 = const ();
StorageDead(_5);
- goto -> bb1;
+ goto -> bb8;
}
bb5: {
StorageLive(_6);
_6 = DFA::C;
_1 = move _6;
_3 = const ();
StorageDead(_6);
- goto -> bb1;
+ goto -> bb9;
}
bb6: {
StorageLive(_7);
_7 = DFA::D;
_1 = move _7;
_3 = const ();
StorageDead(_7);
goto -> bb1;
+ }
+
+ bb7: {
+ _4 = discriminant(_1);
+ goto -> bb4;
+ }
+
+ bb8: {
+ _4 = discriminant(_1);
+ goto -> bb5;
+ }
+
+ bb9: {
+ _4 = discriminant(_1);
+ goto -> bb6;
}
}

87
tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff Normal file
View file

@ -0,0 +1,87 @@
- // MIR for `dfa` before JumpThreading
+ // MIR for `dfa` after JumpThreading
fn dfa() -> () {
let mut _0: ();
let mut _1: DFA;
let mut _2: !;
let mut _3: ();
let mut _4: isize;
let mut _5: DFA;
let mut _6: DFA;
let mut _7: DFA;
let mut _8: !;
scope 1 {
debug state => _1;
}
bb0: {
StorageLive(_1);
_1 = DFA::A;
StorageLive(_2);
- goto -> bb1;
+ goto -> bb7;
}
bb1: {
_4 = discriminant(_1);
- switchInt(move _4) -> [0: bb4, 1: bb5, 2: bb6, 3: bb2, otherwise: bb3];
+ goto -> bb2;
}
bb2: {
_0 = const ();
StorageDead(_2);
StorageDead(_1);
return;
}
bb3: {
unreachable;
}
bb4: {
StorageLive(_5);
_5 = DFA::B;
_1 = move _5;
_3 = const ();
StorageDead(_5);
- goto -> bb1;
+ goto -> bb8;
}
bb5: {
StorageLive(_6);
_6 = DFA::C;
_1 = move _6;
_3 = const ();
StorageDead(_6);
- goto -> bb1;
+ goto -> bb9;
}
bb6: {
StorageLive(_7);
_7 = DFA::D;
_1 = move _7;
_3 = const ();
StorageDead(_7);
goto -> bb1;
+ }
+
+ bb7: {
+ _4 = discriminant(_1);
+ goto -> bb4;
+ }
+
+ bb8: {
+ _4 = discriminant(_1);
+ goto -> bb5;
+ }
+
+ bb9: {
+ _4 = discriminant(_1);
+ goto -> bb6;
}
}

57
tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff Normal file
View file

@ -0,0 +1,57 @@
- // MIR for `disappearing_bb` before JumpThreading
+ // MIR for `disappearing_bb` after JumpThreading
fn disappearing_bb(_1: u8) -> u8 {
let mut _0: u8;
let mut _2: i8;
let mut _3: bool;
let mut _4: bool;
bb0: {
_4 = const false;
_3 = const false;
_4 = const true;
_3 = const true;
switchInt(_1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2];
}
bb1: {
_3 = const false;
- goto -> bb4;
+ goto -> bb9;
}
bb2: {
unreachable;
}
bb3: {
_4 = const false;
goto -> bb4;
}
bb4: {
switchInt(_3) -> [0: bb5, otherwise: bb7];
}
bb5: {
switchInt(_4) -> [0: bb6, otherwise: bb8];
}
bb6: {
return;
}
bb7: {
goto -> bb5;
}
bb8: {
goto -> bb6;
+ }
+
+ bb9: {
+ goto -> bb5;
}
}

57
tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff Normal file
View file

@ -0,0 +1,57 @@
- // MIR for `disappearing_bb` before JumpThreading
+ // MIR for `disappearing_bb` after JumpThreading
fn disappearing_bb(_1: u8) -> u8 {
let mut _0: u8;
let mut _2: i8;
let mut _3: bool;
let mut _4: bool;
bb0: {
_4 = const false;
_3 = const false;
_4 = const true;
_3 = const true;
switchInt(_1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2];
}
bb1: {
_3 = const false;
- goto -> bb4;
+ goto -> bb9;
}
bb2: {
unreachable;
}
bb3: {
_4 = const false;
goto -> bb4;
}
bb4: {
switchInt(_3) -> [0: bb5, otherwise: bb7];
}
bb5: {
switchInt(_4) -> [0: bb6, otherwise: bb8];
}
bb6: {
return;
}
bb7: {
goto -> bb5;
}
bb8: {
goto -> bb6;
+ }
+
+ bb9: {
+ goto -> bb5;
}
}

45
tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff Normal file
View file

@ -0,0 +1,45 @@
- // MIR for `duplicate_chain` before JumpThreading
+ // MIR for `duplicate_chain` after JumpThreading
fn duplicate_chain(_1: bool) -> u8 {
let mut _0: u8;
let mut _2: u8;
let mut _3: i32;
let mut _4: i32;
bb0: {
switchInt(_1) -> [1: bb1, otherwise: bb2];
}
bb1: {
_2 = const 5_u8;
goto -> bb3;
}
bb2: {
_2 = const 5_u8;
goto -> bb3;
}
bb3: {
_3 = const 13_i32;
goto -> bb4;
}
bb4: {
_4 = const 15_i32;
- switchInt(_2) -> [5: bb5, otherwise: bb6];
+ goto -> bb5;
}
bb5: {
_0 = const 7_u8;
return;
}
bb6: {
_0 = const 9_u8;
return;
}
}

45
tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff Normal file
View file

@ -0,0 +1,45 @@
- // MIR for `duplicate_chain` before JumpThreading
+ // MIR for `duplicate_chain` after JumpThreading
fn duplicate_chain(_1: bool) -> u8 {
let mut _0: u8;
let mut _2: u8;
let mut _3: i32;
let mut _4: i32;
bb0: {
switchInt(_1) -> [1: bb1, otherwise: bb2];
}
bb1: {
_2 = const 5_u8;
goto -> bb3;
}
bb2: {
_2 = const 5_u8;
goto -> bb3;
}
bb3: {
_3 = const 13_i32;
goto -> bb4;
}
bb4: {
_4 = const 15_i32;
- switchInt(_2) -> [5: bb5, otherwise: bb6];
+ goto -> bb5;
}
bb5: {
_0 = const 7_u8;
return;
}
bb6: {
_0 = const 9_u8;
return;
}
}

139
tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff Normal file
View file

@ -0,0 +1,139 @@
- // MIR for `identity` before JumpThreading
+ // MIR for `identity` after JumpThreading
fn identity(_1: Result<i32, i32>) -> Result<i32, i32> {
debug x => _1;
let mut _0: std::result::Result<i32, i32>;
let mut _2: i32;
let mut _3: std::ops::ControlFlow<std::result::Result<std::convert::Infallible, i32>, i32>;
let mut _4: std::result::Result<i32, i32>;
let mut _5: isize;
let _6: std::result::Result<std::convert::Infallible, i32>;
let mut _7: !;
let mut _8: std::result::Result<std::convert::Infallible, i32>;
let _9: i32;
scope 1 {
debug residual => _6;
scope 2 {
scope 8 (inlined #[track_caller] <Result<i32, i32> as FromResidual<Result<Infallible, i32>>>::from_residual) {
debug residual => _8;
let _14: i32;
let mut _15: i32;
scope 9 {
debug e => _14;
scope 10 (inlined <i32 as From<i32>>::from) {
debug t => _14;
}
}
}
}
}
scope 3 {
debug val => _9;
scope 4 {
}
}
scope 5 (inlined <Result<i32, i32> as Try>::branch) {
debug self => _4;
let mut _10: isize;
let _11: i32;
let _12: i32;
let mut _13: std::result::Result<std::convert::Infallible, i32>;
scope 6 {
debug v => _11;
}
scope 7 {
debug e => _12;
}
}
bb0: {
StorageLive(_2);
StorageLive(_3);
StorageLive(_4);
_4 = _1;
StorageLive(_10);
StorageLive(_11);
StorageLive(_12);
_10 = discriminant(_4);
switchInt(move _10) -> [0: bb8, 1: bb6, otherwise: bb7];
}
bb1: {
StorageDead(_12);
StorageDead(_11);
StorageDead(_10);
StorageDead(_4);
_5 = discriminant(_3);
- switchInt(move _5) -> [0: bb2, 1: bb4, otherwise: bb3];
+ goto -> bb2;
}
bb2: {
StorageLive(_9);
_9 = ((_3 as Continue).0: i32);
_2 = _9;
StorageDead(_9);
_0 = Result::<i32, i32>::Ok(move _2);
StorageDead(_2);
StorageDead(_3);
goto -> bb5;
}
bb3: {
unreachable;
}
bb4: {
StorageLive(_6);
_6 = ((_3 as Break).0: std::result::Result<std::convert::Infallible, i32>);
StorageLive(_8);
_8 = _6;
StorageLive(_14);
_14 = move ((_8 as Err).0: i32);
StorageLive(_15);
_15 = move _14;
_0 = Result::<i32, i32>::Err(move _15);
StorageDead(_15);
StorageDead(_14);
StorageDead(_8);
StorageDead(_6);
StorageDead(_2);
StorageDead(_3);
goto -> bb5;
}
bb5: {
return;
}
bb6: {
_12 = move ((_4 as Err).0: i32);
StorageLive(_13);
_13 = Result::<Infallible, i32>::Err(move _12);
_3 = ControlFlow::<Result<Infallible, i32>, i32>::Break(move _13);
StorageDead(_13);
- goto -> bb1;
+ goto -> bb9;
}
bb7: {
unreachable;
}
bb8: {
_11 = move ((_4 as Ok).0: i32);
_3 = ControlFlow::<Result<Infallible, i32>, i32>::Continue(move _11);
goto -> bb1;
+ }
+
+ bb9: {
+ StorageDead(_12);
+ StorageDead(_11);
+ StorageDead(_10);
+ StorageDead(_4);
+ _5 = discriminant(_3);
+ goto -> bb4;
}
}

139
tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff Normal file
View file

@ -0,0 +1,139 @@
- // MIR for `identity` before JumpThreading
+ // MIR for `identity` after JumpThreading
fn identity(_1: Result<i32, i32>) -> Result<i32, i32> {
debug x => _1;
let mut _0: std::result::Result<i32, i32>;
let mut _2: i32;
let mut _3: std::ops::ControlFlow<std::result::Result<std::convert::Infallible, i32>, i32>;
let mut _4: std::result::Result<i32, i32>;
let mut _5: isize;
let _6: std::result::Result<std::convert::Infallible, i32>;
let mut _7: !;
let mut _8: std::result::Result<std::convert::Infallible, i32>;
let _9: i32;
scope 1 {
debug residual => _6;
scope 2 {
scope 8 (inlined #[track_caller] <Result<i32, i32> as FromResidual<Result<Infallible, i32>>>::from_residual) {
debug residual => _8;
let _14: i32;
let mut _15: i32;
scope 9 {
debug e => _14;
scope 10 (inlined <i32 as From<i32>>::from) {
debug t => _14;
}
}
}
}
}
scope 3 {
debug val => _9;
scope 4 {
}
}
scope 5 (inlined <Result<i32, i32> as Try>::branch) {
debug self => _4;
let mut _10: isize;
let _11: i32;
let _12: i32;
let mut _13: std::result::Result<std::convert::Infallible, i32>;
scope 6 {
debug v => _11;
}
scope 7 {
debug e => _12;
}
}
bb0: {
StorageLive(_2);
StorageLive(_3);
StorageLive(_4);
_4 = _1;
StorageLive(_10);
StorageLive(_11);
StorageLive(_12);
_10 = discriminant(_4);
switchInt(move _10) -> [0: bb8, 1: bb6, otherwise: bb7];
}
bb1: {
StorageDead(_12);
StorageDead(_11);
StorageDead(_10);
StorageDead(_4);
_5 = discriminant(_3);
- switchInt(move _5) -> [0: bb2, 1: bb4, otherwise: bb3];
+ goto -> bb2;
}
bb2: {
StorageLive(_9);
_9 = ((_3 as Continue).0: i32);
_2 = _9;
StorageDead(_9);
_0 = Result::<i32, i32>::Ok(move _2);
StorageDead(_2);
StorageDead(_3);
goto -> bb5;
}
bb3: {
unreachable;
}
bb4: {
StorageLive(_6);
_6 = ((_3 as Break).0: std::result::Result<std::convert::Infallible, i32>);
StorageLive(_8);
_8 = _6;
StorageLive(_14);
_14 = move ((_8 as Err).0: i32);
StorageLive(_15);
_15 = move _14;
_0 = Result::<i32, i32>::Err(move _15);
StorageDead(_15);
StorageDead(_14);
StorageDead(_8);
StorageDead(_6);
StorageDead(_2);
StorageDead(_3);
goto -> bb5;
}
bb5: {
return;
}
bb6: {
_12 = move ((_4 as Err).0: i32);
StorageLive(_13);
_13 = Result::<Infallible, i32>::Err(move _12);
_3 = ControlFlow::<Result<Infallible, i32>, i32>::Break(move _13);
StorageDead(_13);
- goto -> bb1;
+ goto -> bb9;
}
bb7: {
unreachable;
}
bb8: {
_11 = move ((_4 as Ok).0: i32);
_3 = ControlFlow::<Result<Infallible, i32>, i32>::Continue(move _11);
goto -> bb1;
+ }
+
+ bb9: {
+ StorageDead(_12);
+ StorageDead(_11);
+ StorageDead(_10);
+ StorageDead(_4);
+ _5 = discriminant(_3);
+ goto -> bb4;
}
}

54
tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff Normal file
View file

@ -0,0 +1,54 @@
- // MIR for `multiple_match` before JumpThreading
+ // MIR for `multiple_match` after JumpThreading
fn multiple_match(_1: u8) -> u8 {
let mut _0: u8;
let mut _2: u8;
let mut _3: u8;
bb0: {
switchInt(_1) -> [3: bb1, otherwise: bb2];
}
bb1: {
_2 = _1;
- switchInt(_2) -> [3: bb3, otherwise: bb4];
+ goto -> bb3;
}
bb2: {
_3 = _1;
- switchInt(_3) -> [3: bb5, otherwise: bb6];
+ goto -> bb6;
}
bb3: {
_0 = const 5_u8;
return;
}
bb4: {
_0 = const 7_u8;
return;
}
bb5: {
_0 = const 9_u8;
return;
}
bb6: {
switchInt(_3) -> [1: bb7, otherwise: bb8];
}
bb7: {
_0 = const 9_u8;
return;
}
bb8: {
_0 = const 11_u8;
return;
}
}

54
tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff Normal file
View file

@ -0,0 +1,54 @@
- // MIR for `multiple_match` before JumpThreading
+ // MIR for `multiple_match` after JumpThreading
fn multiple_match(_1: u8) -> u8 {
let mut _0: u8;
let mut _2: u8;
let mut _3: u8;
bb0: {
switchInt(_1) -> [3: bb1, otherwise: bb2];
}
bb1: {
_2 = _1;
- switchInt(_2) -> [3: bb3, otherwise: bb4];
+ goto -> bb3;
}
bb2: {
_3 = _1;
- switchInt(_3) -> [3: bb5, otherwise: bb6];
+ goto -> bb6;
}
bb3: {
_0 = const 5_u8;
return;
}
bb4: {
_0 = const 7_u8;
return;
}
bb5: {
_0 = const 9_u8;
return;
}
bb6: {
switchInt(_3) -> [1: bb7, otherwise: bb8];
}
bb7: {
_0 = const 9_u8;
return;
}
bb8: {
_0 = const 11_u8;
return;
}
}

56
tests/mir-opt/jump_threading.mutable_ref.JumpThreading.panic-abort.diff Normal file
View file

@ -0,0 +1,56 @@
- // MIR for `mutable_ref` before JumpThreading
+ // MIR for `mutable_ref` after JumpThreading
fn mutable_ref() -> bool {
let mut _0: bool;
let mut _1: i32;
let _3: ();
let mut _4: bool;
let mut _5: i32;
scope 1 {
debug x => _1;
let _2: *mut i32;
scope 2 {
debug a => _2;
scope 3 {
}
}
}
bb0: {
StorageLive(_1);
_1 = const 5_i32;
StorageLive(_2);
_2 = &raw mut _1;
_1 = const 7_i32;
StorageLive(_3);
(*_2) = const 8_i32;
_3 = const ();
StorageDead(_3);
StorageLive(_4);
StorageLive(_5);
_5 = _1;
_4 = Eq(move _5, const 7_i32);
switchInt(move _4) -> [0: bb2, otherwise: bb1];
}
bb1: {
StorageDead(_5);
_0 = const true;
goto -> bb3;
}
bb2: {
StorageDead(_5);
_0 = const false;
goto -> bb3;
}
bb3: {
StorageDead(_4);
StorageDead(_2);
StorageDead(_1);
return;
}
}

56
tests/mir-opt/jump_threading.mutable_ref.JumpThreading.panic-unwind.diff Normal file
View file

@ -0,0 +1,56 @@
- // MIR for `mutable_ref` before JumpThreading
+ // MIR for `mutable_ref` after JumpThreading
fn mutable_ref() -> bool {
let mut _0: bool;
let mut _1: i32;
let _3: ();
let mut _4: bool;
let mut _5: i32;
scope 1 {
debug x => _1;
let _2: *mut i32;
scope 2 {
debug a => _2;
scope 3 {
}
}
}
bb0: {
StorageLive(_1);
_1 = const 5_i32;
StorageLive(_2);
_2 = &raw mut _1;
_1 = const 7_i32;
StorageLive(_3);
(*_2) = const 8_i32;
_3 = const ();
StorageDead(_3);
StorageLive(_4);
StorageLive(_5);
_5 = _1;
_4 = Eq(move _5, const 7_i32);
switchInt(move _4) -> [0: bb2, otherwise: bb1];
}
bb1: {
StorageDead(_5);
_0 = const true;
goto -> bb3;
}
bb2: {
StorageDead(_5);
_0 = const false;
goto -> bb3;
}
bb3: {
StorageDead(_4);
StorageDead(_2);
StorageDead(_1);
return;
}
}

26
tests/mir-opt/jump_threading.mutate_discriminant.JumpThreading.panic-abort.diff Normal file
View file

@ -0,0 +1,26 @@
- // MIR for `mutate_discriminant` before JumpThreading
+ // MIR for `mutate_discriminant` after JumpThreading
fn mutate_discriminant() -> u8 {
let mut _0: u8;
let mut _1: std::option::Option<NonZeroUsize>;
let mut _2: isize;
bb0: {
discriminant(_1) = 1;
(((_1 as variant#1).0: NonZeroUsize).0: usize) = const 0_usize;
_2 = discriminant(_1);
switchInt(_2) -> [0: bb1, otherwise: bb2];
}
bb1: {
_0 = const 1_u8;
return;
}
bb2: {
_0 = const 2_u8;
unreachable;
}
}

26
tests/mir-opt/jump_threading.mutate_discriminant.JumpThreading.panic-unwind.diff Normal file
View file

@ -0,0 +1,26 @@
- // MIR for `mutate_discriminant` before JumpThreading
+ // MIR for `mutate_discriminant` after JumpThreading
fn mutate_discriminant() -> u8 {
let mut _0: u8;
let mut _1: std::option::Option<NonZeroUsize>;
let mut _2: isize;
bb0: {
discriminant(_1) = 1;
(((_1 as variant#1).0: NonZeroUsize).0: usize) = const 0_usize;
_2 = discriminant(_1);
switchInt(_2) -> [0: bb1, otherwise: bb2];
}
bb1: {
_0 = const 1_u8;
return;
}
bb2: {
_0 = const 2_u8;
unreachable;
}
}

53
tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff Normal file
View file

@ -0,0 +1,53 @@
- // MIR for `renumbered_bb` before JumpThreading
+ // MIR for `renumbered_bb` after JumpThreading
fn renumbered_bb(_1: bool) -> u8 {
let mut _0: u8;
let mut _2: bool;
let mut _3: bool;
bb0: {
_3 = const false;
switchInt(_1) -> [1: bb1, otherwise: bb2];
}
bb1: {
_2 = const false;
- goto -> bb3;
+ goto -> bb8;
}
bb2: {
_2 = _1;
_3 = _1;
goto -> bb3;
}
bb3: {
switchInt(_2) -> [0: bb4, otherwise: bb5];
}
bb4: {
switchInt(_3) -> [0: bb6, otherwise: bb7];
}
bb5: {
_0 = const 7_u8;
return;
}
bb6: {
_0 = const 9_u8;
return;
}
bb7: {
_0 = const 11_u8;
return;
+ }
+
+ bb8: {
+ goto -> bb4;
}
}

53
tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff Normal file
View file

@ -0,0 +1,53 @@
- // MIR for `renumbered_bb` before JumpThreading
+ // MIR for `renumbered_bb` after JumpThreading
fn renumbered_bb(_1: bool) -> u8 {
let mut _0: u8;
let mut _2: bool;
let mut _3: bool;
bb0: {
_3 = const false;
switchInt(_1) -> [1: bb1, otherwise: bb2];
}
bb1: {
_2 = const false;
- goto -> bb3;
+ goto -> bb8;
}
bb2: {
_2 = _1;
_3 = _1;
goto -> bb3;
}
bb3: {
switchInt(_2) -> [0: bb4, otherwise: bb5];
}
bb4: {
switchInt(_3) -> [0: bb6, otherwise: bb7];
}
bb5: {
_0 = const 7_u8;
return;
}
bb6: {
_0 = const 9_u8;
return;
}
bb7: {
_0 = const 11_u8;
return;
+ }
+
+ bb8: {
+ goto -> bb4;
}
}

291
tests/mir-opt/jump_threading.rs Normal file
View file

@ -0,0 +1,291 @@
// unit-test: JumpThreading
// compile-flags: -Zmir-enable-passes=+Inline
// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
#![feature(control_flow_enum)]
#![feature(try_trait_v2)]
#![feature(custom_mir, core_intrinsics, rustc_attrs)]
use std::intrinsics::mir::*;
use std::ops::ControlFlow;
fn too_complex(x: Result<i32, usize>) -> Option<i32> {
match {
match x {
Ok(v) => ControlFlow::Continue(v),
Err(r) => ControlFlow::Break(r),
}
} {
ControlFlow::Continue(v) => Some(v),
ControlFlow::Break(r) => None,
}
}
fn identity(x: Result<i32, i32>) -> Result<i32, i32> {
Ok(x?)
}
enum DFA {
A,
B,
C,
D,
}
fn dfa() {
let mut state = DFA::A;
loop {
match state {
DFA::A => state = DFA::B,
DFA::B => state = DFA::C,
DFA::C => state = DFA::D,
DFA::D => return,
}
}
}
#[repr(u8)]
enum CustomDiscr {
A = 35,
B = 73,
C = 99,
}
fn custom_discr(x: bool) -> u8 {
match if x { CustomDiscr::A } else { CustomDiscr::B } {
CustomDiscr::A => 5,
_ => 13,
}
}
#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
fn multiple_match(x: u8) -> u8 {
mir!(
{
match x { 3 => bb1, _ => bb2 }
}
bb1 = {
// We know `x == 3`, so we can take `bb3`.
let y = x;
match y { 3 => bb3, _ => bb4 }
}
bb2 = {
// We know `x != 3`, so we can take `bb6`.
let z = x;
match z { 3 => bb5, _ => bb6 }
}
bb3 = {
RET = 5;
Return()
}
bb4 = {
RET = 7;
Return()
}
bb5 = {
RET = 9;
Return()
}
bb6 = {
// We know `z != 3`, so we CANNOT take `bb7`.
match z { 1 => bb7, _ => bb8 }
}
bb7 = {
RET = 9;
Return()
}
bb8 = {
RET = 11;
Return()
}
)
}
#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
fn duplicate_chain(x: bool) -> u8 {
mir!(
let a: u8;
{
match x { true => bb1, _ => bb2 }
}
bb1 = {
a = 5;
Goto(bb3)
}
bb2 = {
a = 5;
Goto(bb3)
}
// Verify that we do not create multiple copied of `bb3`.
bb3 = {
let b = 13;
Goto(bb4)
}
bb4 = {
let c = 15;
match a { 5 => bb5, _ => bb6 }
}
bb5 = {
RET = 7;
Return()
}
bb6 = {
RET = 9;
Return()
}
)
}
#[rustc_layout_scalar_valid_range_start(1)]
#[rustc_nonnull_optimization_guaranteed]
struct NonZeroUsize(usize);
#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
fn mutate_discriminant() -> u8 {
mir!(
let x: Option<NonZeroUsize>;
{
SetDiscriminant(x, 1);
// This assignment overwrites the niche in which the discriminant is stored.
place!(Field(Field(Variant(x, 1), 0), 0)) = 0_usize;
// So we cannot know the value of this discriminant.
let a = Discriminant(x);
match a {
0 => bb1,
_ => bad,
}
}
bb1 = {
RET = 1;
Return()
}
bad = {
RET = 2;
Unreachable()
}
)
}
// Verify that we do not try to reason when there are mutable pointers involved.
fn mutable_ref() -> bool {
let mut x = 5;
let a = std::ptr::addr_of_mut!(x);
x = 7;
unsafe { *a = 8 };
if x == 7 {
true
} else {
false
}
}
#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
fn renumbered_bb(x: bool) -> u8 {
// This function has 2 TOs: 1-3-4 and 0-1-3-4-6.
// We verify that the second TO does not modify 3 once the first has been applied.
mir!(
let a: bool;
let b: bool;
{
b = false;
match x { true => bb1, _ => bb2 }
}
bb1 = {
a = false;
Goto(bb3)
}
bb2 = {
a = x;
b = x;
Goto(bb3)
}
bb3 = {
match a { false => bb4, _ => bb5 }
}
bb4 = {
match b { false => bb6, _ => bb7 }
}
bb5 = {
RET = 7;
Return()
}
bb6 = {
RET = 9;
Return()
}
bb7 = {
RET = 11;
Return()
}
)
}
#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
fn disappearing_bb(x: u8) -> u8 {
// This function has 3 TOs: 1-4-5, 0-1-4-7-5-8 and 3-4-7-5-6
// After applying the first TO, we create bb9 to replace 4, and rename 1-4 edge by 1-9. The
// second TO may try to thread non-existing edge 9-4.
// This test verifies that we preserve semantics by bailing out of this second TO.
mir!(
let _11: i8;
let _12: bool;
let _13: bool;
{
_13 = false;
_12 = false;
_13 = true;
_12 = true;
match x { 0 => bb3, 1 => bb3, 2 => bb1, _ => bb2 }
}
bb1 = {
_12 = false;
Goto(bb4)
}
bb2 = {
Unreachable()
}
bb3 = {
_13 = false;
Goto(bb4)
}
bb4 = {
match _12 { false => bb5, _ => bb7 }
}
bb5 = {
match _13 { false => bb6, _ => bb8 }
}
bb6 = {
Return()
}
bb7 = {
Goto(bb5)
}
bb8 = {
Goto(bb6)
}
)
}
fn main() {
too_complex(Ok(0));
identity(Ok(0));
custom_discr(false);
dfa();
multiple_match(5);
duplicate_chain(false);
mutate_discriminant();
mutable_ref();
renumbered_bb(true);
disappearing_bb(7);
}
// EMIT_MIR jump_threading.too_complex.JumpThreading.diff
// EMIT_MIR jump_threading.identity.JumpThreading.diff
// EMIT_MIR jump_threading.custom_discr.JumpThreading.diff
// EMIT_MIR jump_threading.dfa.JumpThreading.diff
// EMIT_MIR jump_threading.multiple_match.JumpThreading.diff
// EMIT_MIR jump_threading.duplicate_chain.JumpThreading.diff
// EMIT_MIR jump_threading.mutate_discriminant.JumpThreading.diff
// EMIT_MIR jump_threading.mutable_ref.JumpThreading.diff
// EMIT_MIR jump_threading.renumbered_bb.JumpThreading.diff
// EMIT_MIR jump_threading.disappearing_bb.JumpThreading.diff

98
tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff Normal file
View file

@ -0,0 +1,98 @@
- // MIR for `too_complex` before JumpThreading
+ // MIR for `too_complex` after JumpThreading
fn too_complex(_1: Result<i32, usize>) -> Option<i32> {
debug x => _1;
let mut _0: std::option::Option<i32>;
let mut _2: std::ops::ControlFlow<usize, i32>;
let mut _3: isize;
let _4: i32;
let mut _5: i32;
let _6: usize;
let mut _7: usize;
let mut _8: isize;
let _9: i32;
let mut _10: i32;
let _11: usize;
scope 1 {
debug v => _4;
}
scope 2 {
debug r => _6;
}
scope 3 {
debug v => _9;
}
scope 4 {
debug r => _11;
}
bb0: {
StorageLive(_2);
_3 = discriminant(_1);
switchInt(move _3) -> [0: bb3, 1: bb1, otherwise: bb2];
}
bb1: {
StorageLive(_6);
_6 = ((_1 as Err).0: usize);
StorageLive(_7);
_7 = _6;
_2 = ControlFlow::<usize, i32>::Break(move _7);
StorageDead(_7);
StorageDead(_6);
- goto -> bb4;
+ goto -> bb8;
}
bb2: {
unreachable;
}
bb3: {
StorageLive(_4);
_4 = ((_1 as Ok).0: i32);
StorageLive(_5);
_5 = _4;
_2 = ControlFlow::<usize, i32>::Continue(move _5);
StorageDead(_5);
StorageDead(_4);
goto -> bb4;
}
bb4: {
_8 = discriminant(_2);
- switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb2];
+ goto -> bb6;
}
bb5: {
StorageLive(_11);
_11 = ((_2 as Break).0: usize);
_0 = Option::<i32>::None;
StorageDead(_11);
goto -> bb7;
}
bb6: {
StorageLive(_9);
_9 = ((_2 as Continue).0: i32);
StorageLive(_10);
_10 = _9;
_0 = Option::<i32>::Some(move _10);
StorageDead(_10);
StorageDead(_9);
goto -> bb7;
}
bb7: {
StorageDead(_2);
return;
+ }
+
+ bb8: {
+ _8 = discriminant(_2);
+ goto -> bb5;
}
}

98
tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff Normal file
View file

@ -0,0 +1,98 @@
- // MIR for `too_complex` before JumpThreading
+ // MIR for `too_complex` after JumpThreading
fn too_complex(_1: Result<i32, usize>) -> Option<i32> {
debug x => _1;
let mut _0: std::option::Option<i32>;
let mut _2: std::ops::ControlFlow<usize, i32>;
let mut _3: isize;
let _4: i32;
let mut _5: i32;
let _6: usize;
let mut _7: usize;
let mut _8: isize;
let _9: i32;
let mut _10: i32;
let _11: usize;
scope 1 {
debug v => _4;
}
scope 2 {
debug r => _6;
}
scope 3 {
debug v => _9;
}
scope 4 {
debug r => _11;
}
bb0: {
StorageLive(_2);
_3 = discriminant(_1);
switchInt(move _3) -> [0: bb3, 1: bb1, otherwise: bb2];
}
bb1: {
StorageLive(_6);
_6 = ((_1 as Err).0: usize);
StorageLive(_7);
_7 = _6;
_2 = ControlFlow::<usize, i32>::Break(move _7);
StorageDead(_7);
StorageDead(_6);
- goto -> bb4;
+ goto -> bb8;
}
bb2: {
unreachable;
}
bb3: {
StorageLive(_4);
_4 = ((_1 as Ok).0: i32);
StorageLive(_5);
_5 = _4;
_2 = ControlFlow::<usize, i32>::Continue(move _5);
StorageDead(_5);
StorageDead(_4);
goto -> bb4;
}
bb4: {
_8 = discriminant(_2);
- switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb2];
+ goto -> bb6;
}
bb5: {
StorageLive(_11);
_11 = ((_2 as Break).0: usize);
_0 = Option::<i32>::None;
StorageDead(_11);
goto -> bb7;
}
bb6: {
StorageLive(_9);
_9 = ((_2 as Continue).0: i32);
StorageLive(_10);
_10 = _9;
_0 = Option::<i32>::Some(move _10);
StorageDead(_10);
StorageDead(_9);
goto -> bb7;
}
bb7: {
StorageDead(_2);
return;
+ }
+
+ bb8: {
+ _8 = discriminant(_2);
+ goto -> bb5;
}
}