Auto merge of #64949 - nnethercote:avoid-SmallVec-collect, r=zackmdavis

Avoid `SmallVec::collect`

We can get sizeable speed-ups by avoiding `SmallVec::collect` when the number of elements is small.

src/librustc/ty/context.rs

@@ -2868,8 +2868,29 @@ impl<'a, T, R> InternIteratorElement<T, R> for &'a T
 
 impl<T, R, E> InternIteratorElement<T, R> for Result<T, E> {
     type Output = Result<R, E>;
-    fn intern_with<I: Iterator<Item=Self>, F: FnOnce(&[T]) -> R>(iter: I, f: F) -> Self::Output {
-        Ok(f(&iter.collect::<Result<SmallVec<[_; 8]>, _>>()?))
+    fn intern_with<I: Iterator<Item=Self>, F: FnOnce(&[T]) -> R>(mut iter: I, f: F)
+            -> Self::Output {
+        // This code is hot enough that it's worth specializing for the most
+        // common length lists, to avoid the overhead of `SmallVec` creation.
+        // The match arms are in order of frequency. The 1, 2, and 0 cases are
+        // typically hit in ~95% of cases. We assume that if the upper and
+        // lower bounds from `size_hint` agree they are correct.
+        Ok(match iter.size_hint() {
+            (1, Some(1)) => {
+                f(&[iter.next().unwrap()?])
+            }
+            (2, Some(2)) => {
+                let t0 = iter.next().unwrap()?;
+                let t1 = iter.next().unwrap()?;
+                f(&[t0, t1])
+            }
+            (0, Some(0)) => {
+                f(&[])
+            }
+            _ => {
+                f(&iter.collect::<Result<SmallVec<[_; 8]>, _>>()?)
+            }
+        })
     }
 }
 

src/librustc/ty/structural_impls.rs

@@ -1223,9 +1223,22 @@ BraceStructTypeFoldableImpl! {
 
 impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Predicate<'tcx>> {
     fn super_fold_with<F: TypeFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        // This code is hot enough that it's worth specializing for a list of
+        // length 0. (No other length is common enough to be worth singling
+        // out).
+        if self.len() == 0 {
+            self
+        } else {
+            // Don't bother interning if nothing changed, which is the common
+            // case.
             let v = self.iter().map(|p| p.fold_with(folder)).collect::<SmallVec<[_; 8]>>();
+            if v[..] == self[..] {
+                self
+            } else {
                 folder.tcx().intern_predicates(&v)
             }
+        }
+    }
 
     fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool {
         self.iter().any(|p| p.visit_with(visitor))

src/librustc/ty/subst.rs

@@ -402,16 +402,43 @@ impl<'a, 'tcx> InternalSubsts<'tcx> {
 
 impl<'tcx> TypeFoldable<'tcx> for SubstsRef<'tcx> {
     fn super_fold_with<F: TypeFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        // This code is hot enough that it's worth specializing for the most
+        // common length lists, to avoid the overhead of `SmallVec` creation.
+        // The match arms are in order of frequency. The 1, 2, and 0 cases are
+        // typically hit in 90--99.99% of cases. When folding doesn't change
+        // the substs, it's faster to reuse the existing substs rather than
+        // calling `intern_substs`.
+        match self.len() {
+            1 => {
+                let param0 = self[0].fold_with(folder);
+                if param0 == self[0] {
+                    self
+                } else {
+                    folder.tcx().intern_substs(&[param0])
+                }
+            }
+            2 => {
+                let param0 = self[0].fold_with(folder);
+                let param1 = self[1].fold_with(folder);
+                if param0 == self[0] && param1 == self[1] {
+                    self
+                } else {
+                    folder.tcx().intern_substs(&[param0, param1])
+                }
+            }
+            0 => {
+                self
+            }
+            _ => {
                 let params: SmallVec<[_; 8]> = self.iter().map(|k| k.fold_with(folder)).collect();
-
-        // If folding doesn't change the substs, it's faster to avoid
-        // calling `mk_substs` and instead reuse the existing substs.
                 if params[..] == self[..] {
                     self
                 } else {
                     folder.tcx().intern_substs(&params)
                 }
             }
+        }
+    }
 
     fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool {
         self.iter().any(|t| t.visit_with(visitor))