From 07968a001d9e7dfd2417dfb57d3227e395691418 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Mon, 29 Mar 2021 10:45:09 +0200
Subject: [PATCH] Merge commit '0969bc6dde001e01e7e1f58c8ccd7750f8a49ae1' into
 sync_cg_clif-2021-03-29

---
 .github/workflows/bootstrap_rustc.yml | 44 -------------
 .github/workflows/main.yml            | 21 ++++++-
 .github/workflows/rustc.yml           | 82 ++++++++++++++++++++++++
 .vscode/settings.json                 |  2 +-
 Cargo.lock                            | 50 +++++++++------
 Cargo.toml                            |  6 +-
 Readme.md                             | 76 +++++++----------------
 build.sh                              |  7 +++
 build_sysroot/Cargo.lock              | 16 ++---
 build_sysroot/build_sysroot.sh        |  2 +-
 docs/env_vars.md                      |  5 +-
 docs/usage.md                         | 66 ++++++++++++++++++++
 example/mini_core.rs                  |  1 +
 example/mini_core_hello_world.rs      |  9 +--
 prepare.sh                            |  1 -
 rust-toolchain                        |  4 +-
 scripts/cargo.sh                      |  2 +-
 scripts/config.sh                     | 14 +----
 scripts/rustup.sh                     |  2 +-
 scripts/setup_rust_fork.sh            | 68 ++++++++++++++++++++
 scripts/test_bootstrap.sh             | 62 +------------------
 scripts/test_rustc_tests.sh           | 87 ++++++++++++++++++++++++++
 scripts/tests.sh                      | 47 +++++++++-----
 src/abi/comments.rs                   | 27 +++++---
 src/abi/mod.rs                        | 24 +++-----
 src/abi/pass_mode.rs                  |  7 +--
 src/abi/returning.rs                  |  6 +-
 src/allocator.rs                      |  5 +-
 src/base.rs                           | 63 +++++++++----------
 src/codegen_i128.rs                   | 89 +++++++++++++++++++++------
 src/common.rs                         |  3 +-
 src/compiler_builtins.rs              | 41 ++++++++++++
 src/constant.rs                       | 29 +++++----
 src/debuginfo/line_info.rs            |  4 +-
 src/driver/aot.rs                     | 22 +++++--
 src/driver/jit.rs                     | 89 ++++++++++++++++++---------
 src/driver/mod.rs                     |  8 ++-
 src/inline_asm.rs                     | 14 +++--
 src/intrinsics/simd.rs                |  2 +-
 src/lib.rs                            | 39 +++++-------
 src/linkage.rs                        |  2 +
 src/main_shim.rs                      |  9 +--
 src/metadata.rs                       | 72 ++++++++++++++--------
 src/num.rs                            |  8 +--
 src/optimize/stack2reg.rs             | 89 ++++++++++++++-------------
 src/pointer.rs                        |  3 +-
 src/pretty_clif.rs                    | 24 +++++---
 src/trap.rs                           |  3 +-
 src/value_and_place.rs                | 19 ++++--
 49 files changed, 880 insertions(+), 495 deletions(-)
 delete mode 100644 .github/workflows/bootstrap_rustc.yml
 create mode 100644 .github/workflows/rustc.yml
 create mode 100644 docs/usage.md
 create mode 100644 scripts/setup_rust_fork.sh
 create mode 100755 scripts/test_rustc_tests.sh
 create mode 100644 src/compiler_builtins.rs

diff --git a/.github/workflows/bootstrap_rustc.yml b/.github/workflows/bootstrap_rustc.yml
deleted file mode 100644
index 8c94a0aa5e6..00000000000
--- a/.github/workflows/bootstrap_rustc.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-name: Bootstrap rustc using cg_clif
-
-on:
-  - push
-
-jobs:
-  bootstrap_rustc:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Cache cargo installed crates
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-cargo-installed-crates
-
-    - name: Cache cargo registry and index
-      uses: actions/cache@v2
-      with:
-        path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
-    - name: Cache cargo target dir
-      uses: actions/cache@v2
-      with:
-        path: target
-        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
-
-    - name: Prepare dependencies
-      run: |
-        git config --global user.email "user@example.com"
-        git config --global user.name "User"
-        ./prepare.sh
-
-    - name: Test
-      run: |
-        # Enable backtraces for easier debugging
-        export RUST_BACKTRACE=1
-
-        ./scripts/test_bootstrap.sh
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index e6d3375fb1b..2ac516381cf 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -7,11 +7,18 @@ on:
 jobs:
   build:
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
 
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        include:
+          - os: ubuntu-latest
+          - os: macos-latest
+          # cross-compile from Linux to Windows using mingw
+          - os: ubuntu-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-gnu
 
     steps:
     - uses: actions/checkout@v2
@@ -36,6 +43,12 @@ jobs:
         path: target
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
+    - name: Install MinGW toolchain and wine
+      if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      run: |
+        sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
+        rustup target add x86_64-pc-windows-gnu
+
     - name: Prepare dependencies
       run: |
         git config --global user.email "user@example.com"
@@ -43,6 +56,8 @@ jobs:
         ./prepare.sh
 
     - name: Test
+      env:
+        TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
       run: |
         # Enable backtraces for easier debugging
         export RUST_BACKTRACE=1
@@ -51,12 +66,16 @@ jobs:
         export COMPILE_RUNS=2
         export RUN_RUNS=2
 
+        # Enable extra checks
+        export CG_CLIF_ENABLE_VERIFIER=1
+
         ./test.sh
 
     - name: Package prebuilt cg_clif
       run: tar cvfJ cg_clif.tar.xz build
 
     - name: Upload prebuilt cg_clif
+      if: matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
       uses: actions/upload-artifact@v2
       with:
         name: cg_clif-${{ runner.os }}
diff --git a/.github/workflows/rustc.yml b/.github/workflows/rustc.yml
new file mode 100644
index 00000000000..e01a92598ba
--- /dev/null
+++ b/.github/workflows/rustc.yml
@@ -0,0 +1,82 @@
+name: Various rustc tests
+
+on:
+  - push
+
+jobs:
+  bootstrap_rustc:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/bin
+        key: ${{ runner.os }}-cargo-installed-crates
+
+    - name: Cache cargo registry and index
+      uses: actions/cache@v2
+      with:
+        path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./prepare.sh
+
+    - name: Test
+      run: |
+        # Enable backtraces for easier debugging
+        export RUST_BACKTRACE=1
+
+        ./scripts/test_bootstrap.sh
+  rustc_test_suite:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/bin
+        key: ${{ runner.os }}-cargo-installed-crates
+
+    - name: Cache cargo registry and index
+      uses: actions/cache@v2
+      with:
+        path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./prepare.sh
+
+    - name: Test
+      run: |
+        # Enable backtraces for easier debugging
+        export RUST_BACKTRACE=1
+
+        ./scripts/test_rustc_tests.sh
diff --git a/.vscode/settings.json b/.vscode/settings.json
index a13d5931ffa..0cd576e160f 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -2,7 +2,7 @@
     // source for rustc_* is not included in the rust-src component; disable the errors about this
     "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"],
     "rust-analyzer.assist.importMergeBehavior": "last",
-    "rust-analyzer.cargo.loadOutDirsFromCheck": true,
+    "rust-analyzer.cargo.runBuildScripts": true,
     "rust-analyzer.linkedProjects": [
         "./Cargo.toml",
         //"./build_sysroot/sysroot_src/src/libstd/Cargo.toml",
diff --git a/Cargo.lock b/Cargo.lock
index 76d9f0d27ce..3cb67032aaa 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -39,16 +39,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "byteorder",
  "cranelift-bforest",
@@ -65,8 +65,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-codegen-shared",
  "cranelift-entity",
@@ -74,18 +74,18 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -95,8 +95,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-jit"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -113,8 +113,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -125,8 +125,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-codegen",
  "target-lexicon",
@@ -134,8 +134,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+version = "0.72.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -240,6 +240,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "memmap2"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04e3e85b970d650e2ae6d70592474087051c11c54da7f7b4949725c5735fbcc6"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "object"
 version = "0.23.0"
@@ -310,6 +319,7 @@ dependencies = [
  "gimli",
  "indexmap",
  "libloading",
+ "memmap2",
  "object",
  "smallvec",
  "target-lexicon",
diff --git a/Cargo.toml b/Cargo.toml
index 9861af1f8ea..59542c414fa 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -16,12 +16,13 @@ cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch
 cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 target-lexicon = "0.11.0"
 gimli = { version = "0.23.0", default-features = false, features = ["write"]}
-object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] }
+object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
 ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
 indexmap = "1.0.2"
 libloading = { version = "0.6.0", optional = true }
 smallvec = "1.6.1"
+memmap2 = "0.2.1"
 
 # Uncomment to use local checkout of cranelift
 #[patch."https://github.com/bytecodealliance/wasmtime/"]
@@ -75,3 +76,6 @@ debug = false
 [profile.release.package.syn]
 opt-level = 0
 debug = false
+
+[package.metadata.rust-analyzer]
+rustc_private = true
diff --git a/Readme.md b/Readme.md
index 6fa5eebdc2f..ffe1d9a1e65 100644
--- a/Readme.md
+++ b/Readme.md
@@ -34,70 +34,19 @@ rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo bui
 
 Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).
 
-### Cargo
-
 In the directory with your project (where you can do the usual `cargo build`), run:
 
 ```bash
-$ $cg_clif_dir/build/cargo.sh run
+$ $cg_clif_dir/build/cargo.sh build
 ```
 
-This should build and run your project with rustc_codegen_cranelift instead of the usual LLVM backend.
+This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.
 
-### Rustc
-
-> You should prefer using the Cargo method.
-
-```bash
-$ $cg_clif_dir/build/bin/cg_clif my_crate.rs
-```
-
-### Jit mode
-
-In jit mode cg_clif will immediately execute your code without creating an executable file.
-
-> This requires all dependencies to be available as dynamic library.
-> The jit mode will probably need cargo integration to make this possible.
-
-```bash
-$ $cg_clif_dir/build/cargo.sh jit
-```
-
-or
-
-```bash
-$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
-```
-
-There is also an experimental lazy jit mode. In this mode functions are only compiled once they are
-first called. It currently does not work with multi-threaded programs. When a not yet compiled
-function is called from another thread than the main thread, you will get an ICE.
-
-```bash
-$ $cg_clif_dir/build/cargo.sh lazy-jit
-```
-
-### Shell
-
-These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit.
-
-```bash
-function jit_naked() {
-    echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic
-}
-
-function jit() {
-    jit_naked "fn main() { $@ }"
-}
-
-function jit_calc() {
-    jit 'println!("0x{:x}", ' $@ ');';
-}
-```
+For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md).
 
 ## Env vars
 
-[see env_vars.md](docs/env_vars.md)
+See [env_vars.md](docs/env_vars.md) for all env vars used by rustc_codegen_cranelift.
 
 ## Not yet supported
 
@@ -106,3 +55,20 @@ function jit_calc() {
       `llvm_asm!` will remain unimplemented forever. `asm!` doesn't yet support reg classes. You
       have to specify specific registers instead.
 * SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), some basic things work)
+
+## License
+
+Licensed under either of
+
+  * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or
+    http://www.apache.org/licenses/LICENSE-2.0)
+  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+    http://opensource.org/licenses/MIT)
+
+at your option.
+
+### Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally submitted
+for inclusion in the work by you shall be dual licensed as above, without any
+additional terms or conditions.
diff --git a/build.sh b/build.sh
index 090349e54b1..76bc1884334 100755
--- a/build.sh
+++ b/build.sh
@@ -55,6 +55,7 @@ ln target/$CHANNEL/*rustc_codegen_cranelift* "$target_dir"/lib
 ln rust-toolchain scripts/config.sh scripts/cargo.sh "$target_dir"
 
 mkdir -p "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
+mkdir -p "$target_dir/lib/rustlib/$HOST_TRIPLE/lib/"
 if [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then
     cp $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib/*.o "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
 fi
@@ -64,12 +65,18 @@ case "$build_sysroot" in
         ;;
     "llvm")
         cp -r $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib "$target_dir/lib/rustlib/$TARGET_TRIPLE/"
+        if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
+            cp -r $(rustc --print sysroot)/lib/rustlib/$HOST_TRIPLE/lib "$target_dir/lib/rustlib/$HOST_TRIPLE/"
+        fi
         ;;
     "clif")
         echo "[BUILD] sysroot"
         dir=$(pwd)
         cd "$target_dir"
         time "$dir/build_sysroot/build_sysroot.sh"
+        if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
+            time TARGET_TRIPLE="$HOST_TRIPLE" "$dir/build_sysroot/build_sysroot.sh"
+        fi
         cp lib/rustlib/*/lib/libstd-* lib/
         ;;
     *)
diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock
index a7650ab995b..09c5d7590ab 100644
--- a/build_sysroot/Cargo.lock
+++ b/build_sysroot/Cargo.lock
@@ -16,9 +16,9 @@ dependencies = [
 
 [[package]]
 name = "adler"
-version = "0.2.3"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-core",
@@ -110,9 +110,9 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.9.1"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
@@ -132,18 +132,18 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.86"
+version = "0.2.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
+checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7"
 dependencies = [
  "rustc-std-workspace-core",
 ]
 
 [[package]]
 name = "miniz_oxide"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
+checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
 dependencies = [
  "adler",
  "autocfg",
diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh
index 636aa5f3f3d..0354304e55b 100755
--- a/build_sysroot/build_sysroot.sh
+++ b/build_sysroot/build_sysroot.sh
@@ -28,7 +28,7 @@ export __CARGO_DEFAULT_LIB_METADATA="cg_clif"
 if [[ "$1" != "--debug" ]]; then
     sysroot_channel='release'
     # FIXME Enable incremental again once rust-lang/rust#74946 is fixed
-    CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=2" cargo build --target "$TARGET_TRIPLE" --release
+    CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=3" cargo build --target "$TARGET_TRIPLE" --release
 else
     sysroot_channel='debug'
     cargo build --target "$TARGET_TRIPLE"
diff --git a/docs/env_vars.md b/docs/env_vars.md
index f0a0a6ad42e..f7fde1b4f3a 100644
--- a/docs/env_vars.md
+++ b/docs/env_vars.md
@@ -8,5 +8,8 @@
     to make it possible to use incremental mode for all analyses performed by rustc without caching
     object files when their content should have been changed by a change to cg_clif.</dd>
     <dt>CG_CLIF_DISPLAY_CG_TIME</dt>
-    <dd>If "1", display the time it took to perform codegen for a crate</dd>
+    <dd>If "1", display the time it took to perform codegen for a crate.</dd>
+    <dt>CG_CLIF_ENABLE_VERIFIER</dt>
+    <dd>Enable the Cranelift ir verifier for all compilation passes. If not set it will only run once
+    before passing the clif ir to Cranelift for compilation.</dt>
 </dl>
diff --git a/docs/usage.md b/docs/usage.md
new file mode 100644
index 00000000000..3eee3b554e3
--- /dev/null
+++ b/docs/usage.md
@@ -0,0 +1,66 @@
+# Usage
+
+rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects.
+
+Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).
+
+## Cargo
+
+In the directory with your project (where you can do the usual `cargo build`), run:
+
+```bash
+$ $cg_clif_dir/build/cargo.sh build
+```
+
+This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.
+
+## Rustc
+
+> You should prefer using the Cargo method.
+
+```bash
+$ $cg_clif_dir/build/bin/cg_clif my_crate.rs
+```
+
+## Jit mode
+
+In jit mode cg_clif will immediately execute your code without creating an executable file.
+
+> This requires all dependencies to be available as dynamic library.
+> The jit mode will probably need cargo integration to make this possible.
+
+```bash
+$ $cg_clif_dir/build/cargo.sh jit
+```
+
+or
+
+```bash
+$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
+```
+
+There is also an experimental lazy jit mode. In this mode functions are only compiled once they are
+first called. It currently does not work with multi-threaded programs. When a not yet compiled
+function is called from another thread than the main thread, you will get an ICE.
+
+```bash
+$ $cg_clif_dir/build/cargo.sh lazy-jit
+```
+
+## Shell
+
+These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit.
+
+```bash
+function jit_naked() {
+    echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic
+}
+
+function jit() {
+    jit_naked "fn main() { $@ }"
+}
+
+function jit_calc() {
+    jit 'println!("0x{:x}", ' $@ ');';
+}
+```
diff --git a/example/mini_core.rs b/example/mini_core.rs
index 7c6d7fc106d..c4834c80408 100644
--- a/example/mini_core.rs
+++ b/example/mini_core.rs
@@ -621,6 +621,7 @@ struct PanicLocation {
 }
 
 #[no_mangle]
+#[cfg(not(windows))]
 pub fn get_tls() -> u8 {
     #[thread_local]
     static A: u8 = 42;
diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs
index 237f4d11d57..ea37ca98b59 100644
--- a/example/mini_core_hello_world.rs
+++ b/example/mini_core_hello_world.rs
@@ -1,7 +1,4 @@
-#![feature(
-    no_core, start, lang_items, box_syntax, never_type, linkage,
-    extern_types, thread_local
-)]
+#![feature(no_core, lang_items, box_syntax, never_type, linkage, extern_types, thread_local)]
 #![no_core]
 #![allow(dead_code, non_camel_case_types)]
 
@@ -239,7 +236,7 @@ fn main() {
 
     assert_eq!(((|()| 42u8) as fn(()) -> u8)(()), 42);
 
-    #[cfg(not(jit))]
+    #[cfg(not(any(jit, windows)))]
     {
         extern {
             #[linkage = "extern_weak"]
@@ -292,7 +289,7 @@ fn main() {
 
     from_decimal_string();
 
-    #[cfg(not(jit))]
+    #[cfg(not(any(jit, windows)))]
     test_tls();
 
     #[cfg(all(not(jit), target_os = "linux"))]
diff --git a/prepare.sh b/prepare.sh
index ee995ffcfa9..64c097261c9 100755
--- a/prepare.sh
+++ b/prepare.sh
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
-rustup component add rust-src rustc-dev llvm-tools-preview
 ./build_sysroot/prepare_sysroot_src.sh
 cargo install hyperfine || echo "Skipping hyperfine install"
 
diff --git a/rust-toolchain b/rust-toolchain
index 908ca52135b..2917fc7ee39 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1 +1,3 @@
-nightly-2021-03-05
+[toolchain]
+channel = "nightly-2021-03-29"
+components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
diff --git a/scripts/cargo.sh b/scripts/cargo.sh
index 669d2d45b71..1daa5a78f7b 100755
--- a/scripts/cargo.sh
+++ b/scripts/cargo.sh
@@ -4,7 +4,7 @@ dir=$(dirname "$0")
 source "$dir/config.sh"
 
 # read nightly compiler from rust-toolchain file
-TOOLCHAIN=$(cat "$dir/rust-toolchain")
+TOOLCHAIN=$(cat "$dir/rust-toolchain" | grep channel | sed "s/channel = \"\(.*\)\"/\1/")
 
 cmd=$1
 shift || true
diff --git a/scripts/config.sh b/scripts/config.sh
index c2ed2bf256d..99b302ee1d9 100644
--- a/scripts/config.sh
+++ b/scripts/config.sh
@@ -2,15 +2,7 @@
 
 set -e
 
-unamestr=$(uname)
-if [[ "$unamestr" == 'Linux' || "$unamestr" == 'FreeBSD' ]]; then
-   dylib_ext='so'
-elif [[ "$unamestr" == 'Darwin' ]]; then
-   dylib_ext='dylib'
-else
-   echo "Unsupported os"
-   exit 1
-fi
+dylib=$(echo "" | rustc --print file-names --crate-type dylib --crate-name rustc_codegen_cranelift -)
 
 if echo "$RUSTC_WRAPPER" | grep sccache; then
 echo
@@ -24,10 +16,10 @@ dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)
 export RUSTC=$dir"/bin/cg_clif"
 
 export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\
-'-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir
+'-Zcodegen-backend='$dir'/lib/'$dylib' --sysroot '$dir
 
 # FIXME fix `#[linkage = "extern_weak"]` without this
-if [[ "$unamestr" == 'Darwin' ]]; then
+if [[ "$(uname)" == 'Darwin' ]]; then
    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi
 
diff --git a/scripts/rustup.sh b/scripts/rustup.sh
index 694945a87c2..fa7557653d8 100755
--- a/scripts/rustup.sh
+++ b/scripts/rustup.sh
@@ -8,7 +8,7 @@ case $1 in
 
         echo "=> Installing new nightly"
         rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists
-        echo "nightly-${TOOLCHAIN}" > rust-toolchain
+        sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain
         rustup component add rustfmt || true
 
         echo "=> Uninstalling all old nighlies"
diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh
new file mode 100644
index 00000000000..e8bedf625f7
--- /dev/null
+++ b/scripts/setup_rust_fork.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+set -e
+
+./build.sh
+source build/config.sh
+
+echo "[SETUP] Rust fork"
+git clone https://github.com/rust-lang/rust.git || true
+pushd rust
+git fetch
+git checkout -- .
+git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
+
+git apply - <<EOF
+diff --git a/Cargo.toml b/Cargo.toml
+index 5bd1147cad5..10d68a2ff14 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
+ rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
+ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
+
++compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
++
+ [patch."https://github.com/rust-lang/rust-clippy"]
+ clippy_lints = { path = "src/tools/clippy/clippy_lints" }
+diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
+index 23e689fcae7..5f077b765b6 100644
+--- a/compiler/rustc_data_structures/Cargo.toml
++++ b/compiler/rustc_data_structures/Cargo.toml
+@@ -32,7 +32,6 @@ tempfile = "3.0.5"
+
+ [dependencies.parking_lot]
+ version = "0.11"
+-features = ["nightly"]
+
+ [target.'cfg(windows)'.dependencies]
+ winapi = { version = "0.3", features = ["fileapi", "psapi"] }
+diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
+index d95b5b7f17f..00b6f0e3635 100644
+--- a/library/alloc/Cargo.toml
++++ b/library/alloc/Cargo.toml
+@@ -8,7 +8,7 @@ edition = "2018"
+
+ [dependencies]
+ core = { path = "../core" }
+-compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
++compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
+
+ [dev-dependencies]
+ rand = "0.7"
+EOF
+
+cat > config.toml <<EOF
+[llvm]
+ninja = false
+
+[build]
+rustc = "$(pwd)/../build/bin/cg_clif"
+cargo = "$(rustup which cargo)"
+full-bootstrap = true
+local-rebuild = true
+
+[rust]
+codegen-backends = ["cranelift"]
+deny-warnings = false
+EOF
+popd
diff --git a/scripts/test_bootstrap.sh b/scripts/test_bootstrap.sh
index 6473c6ad67d..791d457993d 100755
--- a/scripts/test_bootstrap.sh
+++ b/scripts/test_bootstrap.sh
@@ -3,70 +3,10 @@ set -e
 
 cd "$(dirname "$0")/../"
 
-./build.sh
-source build/config.sh
+source ./scripts/setup_rust_fork.sh
 
 echo "[TEST] Bootstrap of rustc"
-git clone https://github.com/rust-lang/rust.git || true
 pushd rust
-git fetch
-git checkout -- .
-git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
-
-git apply - <<EOF
-diff --git a/Cargo.toml b/Cargo.toml
-index 5bd1147cad5..10d68a2ff14 100644
---- a/Cargo.toml
-+++ b/Cargo.toml
-@@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
- rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
- rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
- 
-+compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
-+
- [patch."https://github.com/rust-lang/rust-clippy"]
- clippy_lints = { path = "src/tools/clippy/clippy_lints" }
-diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
-index 23e689fcae7..5f077b765b6 100644
---- a/compiler/rustc_data_structures/Cargo.toml
-+++ b/compiler/rustc_data_structures/Cargo.toml
-@@ -32,7 +32,6 @@ tempfile = "3.0.5"
-
- [dependencies.parking_lot]
- version = "0.11"
--features = ["nightly"]
-
- [target.'cfg(windows)'.dependencies]
- winapi = { version = "0.3", features = ["fileapi", "psapi"] }
-diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
-index d95b5b7f17f..00b6f0e3635 100644
---- a/library/alloc/Cargo.toml
-+++ b/library/alloc/Cargo.toml
-@@ -8,7 +8,7 @@ edition = "2018"
- 
- [dependencies]
- core = { path = "../core" }
--compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
-+compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
- 
- [dev-dependencies]
- rand = "0.7"
-EOF
-
-cat > config.toml <<EOF
-[llvm]
-ninja = false
-
-[build]
-rustc = "$(pwd)/../build/bin/cg_clif"
-cargo = "$(rustup which cargo)"
-full-bootstrap = true
-local-rebuild = true
-
-[rust]
-codegen-backends = ["cranelift"]
-EOF
-
 rm -r compiler/rustc_codegen_cranelift/{Cargo.*,src}
 cp ../Cargo.* compiler/rustc_codegen_cranelift/
 cp -r ../src compiler/rustc_codegen_cranelift/src
diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh
new file mode 100755
index 00000000000..fbc3feceec7
--- /dev/null
+++ b/scripts/test_rustc_tests.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+set -e
+
+cd $(dirname "$0")/../
+
+source ./scripts/setup_rust_fork.sh
+
+echo "[TEST] Test suite of rustc"
+pushd rust
+
+cargo install ripgrep
+
+rm -r src/test/ui/{extern/,panics/,unsized-locals/,thinlto/,simd*,*lto*.rs,linkage*,unwind-*.rs} || true
+for test in $(rg --files-with-matches "asm!|catch_unwind|should_panic|lto" src/test/ui); do
+  rm $test
+done
+
+for test in $(rg -i --files-with-matches "//(\[\w+\])?~|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" src/test/ui); do
+  rm $test
+done
+
+git checkout -- src/test/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
+
+# these all depend on unwinding support
+rm src/test/ui/backtrace.rs
+rm src/test/ui/array-slice-vec/box-of-array-of-drop-*.rs
+rm src/test/ui/array-slice-vec/slice-panic-*.rs
+rm src/test/ui/array-slice-vec/nested-vec-3.rs
+rm src/test/ui/cleanup-rvalue-temp-during-incomplete-alloc.rs
+rm src/test/ui/issues/issue-26655.rs
+rm src/test/ui/issues/issue-29485.rs
+rm src/test/ui/issues/issue-30018-panic.rs
+rm src/test/ui/multi-panic.rs
+rm src/test/ui/sepcomp/sepcomp-unwind.rs
+rm src/test/ui/structs-enums/unit-like-struct-drop-run.rs
+rm src/test/ui/terminate-in-initializer.rs
+rm src/test/ui/threads-sendsync/task-stderr.rs
+rm src/test/ui/numbers-arithmetic/int-abs-overflow.rs
+rm src/test/ui/drop/drop-trait-enum.rs
+rm src/test/ui/numbers-arithmetic/issue-8460.rs
+
+rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations
+rm src/test/ui/init-large-type.rs # same
+rm src/test/ui/sse2.rs # cpuid not supported, so sse2 not detected
+rm src/test/ui/issues/issue-33992.rs # unsupported linkages
+rm src/test/ui/issues/issue-51947.rs # same
+rm src/test/ui/numbers-arithmetic/saturating-float-casts.rs # intrinsic gives different but valid result
+rm src/test/ui/mir/mir_misc_casts.rs # depends on deduplication of constants
+rm src/test/ui/mir/mir_raw_fat_ptr.rs # same
+rm src/test/ui/async-await/async-fn-size-moved-locals.rs # -Cpanic=abort shrinks some generator by one byte
+rm src/test/ui/async-await/async-fn-size-uninit-locals.rs # same
+rm src/test/ui/generator/size-moved-locals.rs # same
+rm src/test/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment
+rm src/test/ui/test-attrs/test-fn-signature-verification-for-explicit-return-type.rs # "Cannot run dynamic test fn out-of-process"
+rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and nearbyintf64 intrinsics
+
+rm src/test/incremental/hashes/inline_asm.rs # inline asm
+rm src/test/incremental/issue-72386.rs # same
+rm src/test/incremental/change_crate_dep_kind.rs # requires -Cpanic=unwind
+rm src/test/incremental/issue-49482.rs # same
+rm src/test/incremental/issue-54059.rs # same
+rm src/test/incremental/lto.rs # requires lto
+
+rm src/test/pretty/asm.rs # inline asm
+rm src/test/pretty/raw-str-nonexpr.rs # same
+
+rm -r src/test/run-pass-valgrind/unsized-locals
+
+rm src/test/ui/json-bom-plus-crlf-multifile.rs # differing warning
+rm src/test/ui/json-bom-plus-crlf.rs # same
+rm src/test/ui/type-alias-impl-trait/cross_crate_ice*.rs # requires removed aux dep
+
+rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition
+rm src/test/ui/cfg/cfg-panic.rs
+rm src/test/ui/default-alloc-error-hook.rs
+rm -r src/test/ui/hygiene/
+
+rm -r src/test/ui/polymorphization/ # polymorphization not yet supported
+rm src/test/codegen-units/polymorphization/unused_type_parameters.rs # same
+
+rm -r src/test/run-make/fmt-write-bloat/ # tests an optimization
+rm src/test/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs
+rm src/test/ui/abi/variadic-ffi.rs # requires callee side vararg support
+
+echo "[TEST] rustc test suite"
+RUST_TEST_NOCAPTURE=1 COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 src/test/{codegen-units,run-make,run-pass-valgrind,ui}
+popd
diff --git a/scripts/tests.sh b/scripts/tests.sh
index f9a9fb091fb..3afcea8f06b 100755
--- a/scripts/tests.sh
+++ b/scripts/tests.sh
@@ -71,14 +71,20 @@ function base_sysroot_tests() {
     echo "[AOT] mod_bench"
     $MY_RUSTC example/mod_bench.rs --crate-type bin --target "$TARGET_TRIPLE"
     $RUN_WRAPPER ./target/out/mod_bench
-
-    pushd rand
-    rm -r ./target || true
-    ../build/cargo.sh test --workspace
-    popd
 }
 
 function extended_sysroot_tests() {
+    pushd rand
+    cargo clean
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        echo "[TEST] rust-random/rand"
+        ../build/cargo.sh test --workspace
+    else
+        echo "[AOT] rust-random/rand"
+        ../build/cargo.sh build --workspace --target $TARGET_TRIPLE --tests
+    fi
+    popd
+
     pushd simple-raytracer
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
         echo "[BENCH COMPILE] ebobby/simple-raytracer"
@@ -92,27 +98,40 @@ function extended_sysroot_tests() {
     else
         echo "[BENCH COMPILE] ebobby/simple-raytracer (skipped)"
         echo "[COMPILE] ebobby/simple-raytracer"
-        ../cargo.sh build
+        ../build/cargo.sh build --target $TARGET_TRIPLE
         echo "[BENCH RUN] ebobby/simple-raytracer (skipped)"
     fi
     popd
 
     pushd build_sysroot/sysroot_src/library/core/tests
     echo "[TEST] libcore"
-    rm -r ./target || true
-    ../../../../../build/cargo.sh test
+    cargo clean
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        ../../../../../build/cargo.sh test
+    else
+        ../../../../../build/cargo.sh build --target $TARGET_TRIPLE --tests
+    fi
     popd
 
     pushd regex
     echo "[TEST] rust-lang/regex example shootout-regex-dna"
-    ../build/cargo.sh clean
+    cargo clean
     # Make sure `[codegen mono items] start` doesn't poison the diff
-    ../build/cargo.sh build --example shootout-regex-dna
-    cat examples/regexdna-input.txt | ../build/cargo.sh run --example shootout-regex-dna | grep -v "Spawned thread" > res.txt
-    diff -u res.txt examples/regexdna-output.txt
+    ../build/cargo.sh build --example shootout-regex-dna --target $TARGET_TRIPLE
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        cat examples/regexdna-input.txt \
+            | ../build/cargo.sh run --example shootout-regex-dna --target $TARGET_TRIPLE \
+            | grep -v "Spawned thread" > res.txt
+        diff -u res.txt examples/regexdna-output.txt
+    fi
 
-    echo "[TEST] rust-lang/regex tests"
-    ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        echo "[TEST] rust-lang/regex tests"
+        ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+    else
+        echo "[AOT] rust-lang/regex tests"
+        ../build/cargo.sh build --tests --target $TARGET_TRIPLE
+    fi
     popd
 }
 
diff --git a/src/abi/comments.rs b/src/abi/comments.rs
index c3cf90e1e70..5fbaed7283a 100644
--- a/src/abi/comments.rs
+++ b/src/abi/comments.rs
@@ -11,9 +11,11 @@ use cranelift_codegen::entity::EntityRef;
 use crate::prelude::*;
 
 pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
-    fx.add_global_comment(
-        "kind  loc.idx   param    pass mode                            ty".to_string(),
-    );
+    if fx.clif_comments.enabled() {
+        fx.add_global_comment(
+            "kind  loc.idx   param    pass mode                            ty".to_string(),
+        );
+    }
 }
 
 pub(super) fn add_arg_comment<'tcx>(
@@ -25,6 +27,10 @@ pub(super) fn add_arg_comment<'tcx>(
     arg_abi_mode: PassMode,
     arg_layout: TyAndLayout<'tcx>,
 ) {
+    if !fx.clif_comments.enabled() {
+        return;
+    }
+
     let local = if let Some(local) = local {
         Cow::Owned(format!("{:?}", local))
     } else {
@@ -59,10 +65,12 @@ pub(super) fn add_arg_comment<'tcx>(
 }
 
 pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
-    fx.add_global_comment(String::new());
-    fx.add_global_comment(
-        "kind  local ty                              size align (abi,pref)".to_string(),
-    );
+    if fx.clif_comments.enabled() {
+        fx.add_global_comment(String::new());
+        fx.add_global_comment(
+            "kind  local ty                              size align (abi,pref)".to_string(),
+        );
+    }
 }
 
 pub(super) fn add_local_place_comments<'tcx>(
@@ -70,6 +78,9 @@ pub(super) fn add_local_place_comments<'tcx>(
     place: CPlace<'tcx>,
     local: Local,
 ) {
+    if !fx.clif_comments.enabled() {
+        return;
+    }
     let TyAndLayout { ty, layout } = place.layout();
     let rustc_target::abi::Layout { size, align, abi: _, variants: _, fields: _, largest_niche: _ } =
         layout;
@@ -90,7 +101,7 @@ pub(super) fn add_local_place_comments<'tcx>(
             } else {
                 Cow::Borrowed("")
             };
-            match ptr.base_and_offset() {
+            match ptr.debug_base_and_offset() {
                 (crate::pointer::PointerBase::Addr(addr), offset) => {
                     ("reuse", format!("storage={}{}{}", addr, offset, meta).into())
                 }
diff --git a/src/abi/mod.rs b/src/abi/mod.rs
index b158d73f3a1..0e7829eaa26 100644
--- a/src/abi/mod.rs
+++ b/src/abi/mod.rs
@@ -1,6 +1,5 @@
 //! Handling of everything related to the calling convention. Also fills `fx.local_map`.
 
-#[cfg(debug_assertions)]
 mod comments;
 mod pass_mode;
 mod returning;
@@ -75,8 +74,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         let func_id = import_function(self.tcx, self.cx.module, inst);
         let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
 
-        #[cfg(debug_assertions)]
-        self.add_comment(func_ref, format!("{:?}", inst));
+        if self.clif_comments.enabled() {
+            self.add_comment(func_ref, format!("{:?}", inst));
+        }
 
         func_ref
     }
@@ -92,8 +92,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap();
         let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
         let call_inst = self.bcx.ins().call(func_ref, args);
-        #[cfg(debug_assertions)]
-        {
+        if self.clif_comments.enabled() {
             self.add_comment(call_inst, format!("easy_call {}", name));
         }
         let results = self.bcx.inst_results(call_inst);
@@ -149,7 +148,6 @@ fn make_local_place<'tcx>(
         CPlace::new_stack_slot(fx, layout)
     };
 
-    #[cfg(debug_assertions)]
     self::comments::add_local_place_comments(fx, place, local);
 
     place
@@ -163,7 +161,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
 
     let ssa_analyzed = crate::analyze::analyze(fx);
 
-    #[cfg(debug_assertions)]
     self::comments::add_args_header_comment(fx);
 
     let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter();
@@ -228,7 +225,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
     fx.fn_abi = Some(fn_abi);
     assert!(block_params_iter.next().is_none(), "arg_value left behind");
 
-    #[cfg(debug_assertions)]
     self::comments::add_locals_header_comment(fx);
 
     for (local, arg_kind, ty) in func_params {
@@ -256,7 +252,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
                         CPlace::for_ptr(addr, val.layout())
                     };
 
-                    #[cfg(debug_assertions)]
                     self::comments::add_local_place_comments(fx, place, local);
 
                     assert_eq!(fx.local_map.push(place), local);
@@ -392,8 +387,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     let (func_ref, first_arg) = match instance {
         // Trait object call
         Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => {
-            #[cfg(debug_assertions)]
-            {
+            if fx.clif_comments.enabled() {
                 let nop_inst = fx.bcx.ins().nop();
                 fx.add_comment(
                     nop_inst,
@@ -414,8 +408,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
         // Indirect call
         None => {
-            #[cfg(debug_assertions)]
-            {
+            if fx.clif_comments.enabled() {
                 let nop_inst = fx.bcx.ins().nop();
                 fx.add_comment(nop_inst, "indirect call");
             }
@@ -477,10 +470,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     // FIXME find a cleaner way to support varargs
     if fn_sig.c_variadic {
         if !matches!(fn_sig.abi, Abi::C { .. }) {
-            fx.tcx.sess.span_fatal(
-                span,
-                &format!("Variadic call for non-C abi {:?}", fn_sig.abi),
-            );
+            fx.tcx.sess.span_fatal(span, &format!("Variadic call for non-C abi {:?}", fn_sig.abi));
         }
         let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap();
         let abi_params = call_args
diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs
index d58f952f53c..7c275965199 100644
--- a/src/abi/pass_mode.rs
+++ b/src/abi/pass_mode.rs
@@ -208,7 +208,7 @@ pub(super) fn from_casted_value<'tcx>(
     });
     let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
     let mut offset = 0;
-    let mut block_params_iter = block_params.into_iter().copied();
+    let mut block_params_iter = block_params.iter().copied();
     for param in abi_params {
         let val = ptr.offset_i64(fx, offset).store(
             fx,
@@ -248,8 +248,8 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
 /// as necessary.
 pub(super) fn cvalue_for_param<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option<mir::Local>,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option<usize>,
+    local: Option<mir::Local>,
+    local_field: Option<usize>,
     arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
     block_params_iter: &mut impl Iterator<Item = Value>,
 ) -> Option<CValue<'tcx>> {
@@ -263,7 +263,6 @@ pub(super) fn cvalue_for_param<'tcx>(
         })
         .collect::<SmallVec<[_; 2]>>();
 
-    #[cfg(debug_assertions)]
     crate::abi::comments::add_arg_comment(
         fx,
         "arg",
diff --git a/src/abi/returning.rs b/src/abi/returning.rs
index 9fa066df69b..e1c53224b4f 100644
--- a/src/abi/returning.rs
+++ b/src/abi/returning.rs
@@ -84,10 +84,6 @@ pub(super) fn codegen_return_param<'tcx>(
         }
     };
 
-    #[cfg(not(debug_assertions))]
-    let _ = ret_param;
-
-    #[cfg(debug_assertions)]
     crate::abi::comments::add_arg_comment(
         fx,
         "ret",
@@ -146,7 +142,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx, T>(
                 let results = fx
                     .bcx
                     .inst_results(call_inst)
-                    .into_iter()
+                    .iter()
                     .copied()
                     .collect::<SmallVec<[Value; 2]>>();
                 let result =
diff --git a/src/allocator.rs b/src/allocator.rs
index efb64233ef2..f60645a9f97 100644
--- a/src/allocator.rs
+++ b/src/allocator.rs
@@ -3,6 +3,7 @@
 
 use crate::prelude::*;
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
 use rustc_span::symbol::sym;
 
@@ -92,7 +93,7 @@ fn codegen_inner(
             bcx.finalize();
         }
         module
-            .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
+            .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
             .unwrap();
         unwind_context.add_function(func_id, &ctx, module.isa());
     }
@@ -132,7 +133,7 @@ fn codegen_inner(
         bcx.finalize();
     }
     module
-        .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
+        .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
         .unwrap();
     unwind_context.add_function(func_id, &ctx, module.isa());
 }
diff --git a/src/base.rs b/src/base.rs
index 8b5ae9e0541..b34a29c25b9 100644
--- a/src/base.rs
+++ b/src/base.rs
@@ -1,5 +1,6 @@
 //! Codegen of a single function
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::layout::FnAbiExt;
@@ -7,11 +8,7 @@ use rustc_target::abi::call::FnAbi;
 
 use crate::prelude::*;
 
-pub(crate) fn codegen_fn<'tcx>(
-    cx: &mut crate::CodegenCx<'_, 'tcx>,
-    instance: Instance<'tcx>,
-    linkage: Linkage,
-) {
+pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) {
     let tcx = cx.tcx;
 
     let _inst_guard =
@@ -23,7 +20,7 @@ pub(crate) fn codegen_fn<'tcx>(
     // Declare function
     let name = tcx.symbol_name(instance).name.to_string();
     let sig = get_function_sig(tcx, cx.module.isa().triple(), instance);
-    let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap();
+    let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap();
 
     cx.cached_context.clear();
 
@@ -131,7 +128,7 @@ pub(crate) fn codegen_fn<'tcx>(
     let module = &mut cx.module;
     tcx.sess.time("define function", || {
         module
-            .define_function(func_id, context, &mut cranelift_codegen::binemit::NullTrapSink {})
+            .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {})
             .unwrap()
     });
 
@@ -219,8 +216,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
             codegen_stmt(fx, block, stmt);
         }
 
-        #[cfg(debug_assertions)]
-        {
+        if fx.clif_comments.enabled() {
             let mut terminator_head = "\n".to_string();
             bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
             let inst = fx.bcx.func.layout.last_inst(block).unwrap();
@@ -433,12 +429,14 @@ fn codegen_stmt<'tcx>(
 
     fx.set_debug_loc(stmt.source_info);
 
-    #[cfg(false_debug_assertions)]
+    #[cfg(disabled)]
     match &stmt.kind {
         StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
         _ => {
-            let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
-            fx.add_comment(inst, format!("{:?}", stmt));
+            if fx.clif_comments.enabled() {
+                let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
+                fx.add_comment(inst, format!("{:?}", stmt));
+            }
         }
     }
 
@@ -464,16 +462,16 @@ fn codegen_stmt<'tcx>(
                     let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout());
                     lval.write_cvalue(fx, val);
                 }
-                Rvalue::BinaryOp(bin_op, box (ref lhs, ref rhs)) => {
-                    let lhs = codegen_operand(fx, lhs);
-                    let rhs = codegen_operand(fx, rhs);
+                Rvalue::BinaryOp(bin_op, ref lhs_rhs) => {
+                    let lhs = codegen_operand(fx, &lhs_rhs.0);
+                    let rhs = codegen_operand(fx, &lhs_rhs.1);
 
                     let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs);
                     lval.write_cvalue(fx, res);
                 }
-                Rvalue::CheckedBinaryOp(bin_op, box (ref lhs, ref rhs)) => {
-                    let lhs = codegen_operand(fx, lhs);
-                    let rhs = codegen_operand(fx, rhs);
+                Rvalue::CheckedBinaryOp(bin_op, ref lhs_rhs) => {
+                    let lhs = codegen_operand(fx, &lhs_rhs.0);
+                    let rhs = codegen_operand(fx, &lhs_rhs.1);
 
                     let res = if !fx.tcx.sess.overflow_checks() {
                         let val =
@@ -659,7 +657,9 @@ fn codegen_stmt<'tcx>(
                         .val
                         .try_to_bits(fx.tcx.data_layout.pointer_size)
                         .unwrap();
-                    if fx.clif_type(operand.layout().ty) == Some(types::I8) {
+                    if operand.layout().size.bytes() == 0 {
+                        // Do nothing for ZST's
+                    } else if fx.clif_type(operand.layout().ty) == Some(types::I8) {
                         let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64);
                         // FIXME use emit_small_memset where possible
                         let addr = lval.to_ptr().get_addr(fx);
@@ -832,25 +832,18 @@ fn codegen_stmt<'tcx>(
             }
         }
         StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
-        StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
-          src,
-          dst,
-          count,
-        }) => {
-            let dst = codegen_operand(fx, dst);
+        StatementKind::CopyNonOverlapping(inner) => {
+            let dst = codegen_operand(fx, &inner.dst);
             let pointee = dst
-              .layout()
-              .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
-              .expect("Expected pointer");
+                .layout()
+                .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
+                .expect("Expected pointer");
             let dst = dst.load_scalar(fx);
-            let src = codegen_operand(fx, src).load_scalar(fx);
-            let count = codegen_operand(fx, count).load_scalar(fx);
+            let src = codegen_operand(fx, &inner.src).load_scalar(fx);
+            let count = codegen_operand(fx, &inner.count).load_scalar(fx);
             let elem_size: u64 = pointee.size.bytes();
-            let bytes = if elem_size != 1 {
-               fx.bcx.ins().imul_imm(count, elem_size as i64)
-            } else {
-               count
-            };
+            let bytes =
+                if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
             fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes);
         }
     }
diff --git a/src/codegen_i128.rs b/src/codegen_i128.rs
index ae75e6508cb..ffe1922ab90 100644
--- a/src/codegen_i128.rs
+++ b/src/codegen_i128.rs
@@ -32,18 +32,56 @@ pub(crate) fn maybe_codegen<'tcx>(
         BinOp::Add | BinOp::Sub if !checked => None,
         BinOp::Mul if !checked => {
             let val_ty = if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-            Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
+            if fx.tcx.sess.target.is_like_windows {
+                let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                let args =
+                    [ret_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                fx.lib_call(
+                    "__multi3",
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                    ],
+                    vec![],
+                    &args,
+                );
+                Some(ret_place.to_cvalue(fx))
+            } else {
+                Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
+            }
         }
         BinOp::Add | BinOp::Sub | BinOp::Mul => {
             assert!(checked);
             let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
             let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty));
-            let param_types = vec![
-                AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
-                AbiParam::new(types::I128),
-                AbiParam::new(types::I128),
-            ];
-            let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)];
+            let (param_types, args) = if fx.tcx.sess.target.is_like_windows {
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                (
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                    ],
+                    [out_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)],
+                )
+            } else {
+                (
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(types::I128),
+                        AbiParam::new(types::I128),
+                    ],
+                    [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)],
+                )
+            };
             let name = match (bin_op, is_signed) {
                 (BinOp::Add, false) => "__rust_u128_addo",
                 (BinOp::Add, true) => "__rust_i128_addo",
@@ -57,20 +95,33 @@ pub(crate) fn maybe_codegen<'tcx>(
             Some(out_place.to_cvalue(fx))
         }
         BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
-        BinOp::Div => {
+        BinOp::Div | BinOp::Rem => {
             assert!(!checked);
-            if is_signed {
-                Some(fx.easy_call("__divti3", &[lhs, rhs], fx.tcx.types.i128))
+            let name = match (bin_op, is_signed) {
+                (BinOp::Div, false) => "__udivti3",
+                (BinOp::Div, true) => "__divti3",
+                (BinOp::Rem, false) => "__umodti3",
+                (BinOp::Rem, true) => "__modti3",
+                _ => unreachable!(),
+            };
+            if fx.tcx.sess.target.is_like_windows {
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                let args = [lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                let ret = fx.lib_call(
+                    name,
+                    vec![AbiParam::new(pointer_ty(fx.tcx)), AbiParam::new(pointer_ty(fx.tcx))],
+                    vec![AbiParam::new(types::I64X2)],
+                    &args,
+                )[0];
+                // FIXME use bitcast instead of store to get from i64x2 to i128
+                let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
+                ret_place.to_ptr().store(fx, ret, MemFlags::trusted());
+                Some(ret_place.to_cvalue(fx))
             } else {
-                Some(fx.easy_call("__udivti3", &[lhs, rhs], fx.tcx.types.u128))
-            }
-        }
-        BinOp::Rem => {
-            assert!(!checked);
-            if is_signed {
-                Some(fx.easy_call("__modti3", &[lhs, rhs], fx.tcx.types.i128))
-            } else {
-                Some(fx.easy_call("__umodti3", &[lhs, rhs], fx.tcx.types.u128))
+                Some(fx.easy_call(name, &[lhs, rhs], lhs.layout().ty))
             }
         }
         BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => {
diff --git a/src/common.rs b/src/common.rs
index 6a4a6744a5c..b5874f62535 100644
--- a/src/common.rs
+++ b/src/common.rs
@@ -361,8 +361,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         let _ = self.cx.module.define_data(msg_id, &data_ctx);
 
         let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func);
-        #[cfg(debug_assertions)]
-        {
+        if self.clif_comments.enabled() {
             self.add_comment(local_msg_id, msg);
         }
         self.bcx.ins().global_value(self.pointer_type, local_msg_id)
diff --git a/src/compiler_builtins.rs b/src/compiler_builtins.rs
new file mode 100644
index 00000000000..177f850afb3
--- /dev/null
+++ b/src/compiler_builtins.rs
@@ -0,0 +1,41 @@
+macro builtin_functions($register:ident; $(fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;)*) {
+    #[cfg(feature = "jit")]
+    #[allow(improper_ctypes)]
+    extern "C" {
+        $(fn $name($($arg_name: $arg_ty),*) -> $ret_ty;)*
+    }
+
+    #[cfg(feature = "jit")]
+    pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) {
+        for &(name, val) in &[$((stringify!($name), $name as *const u8)),*] {
+            builder.symbol(name, val);
+        }
+    }
+}
+
+builtin_functions! {
+    register_functions_for_jit;
+
+    // integers
+    fn __multi3(a: i128, b: i128) -> i128;
+    fn __udivti3(n: u128, d: u128) -> u128;
+    fn __divti3(n: i128, d: i128) -> i128;
+    fn __umodti3(n: u128, d: u128) -> u128;
+    fn __modti3(n: i128, d: i128) -> i128;
+    fn __rust_u128_addo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_addo(a: i128, b: i128) -> (i128, bool);
+    fn __rust_u128_subo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_subo(a: i128, b: i128) -> (i128, bool);
+    fn __rust_u128_mulo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_mulo(a: i128, b: i128) -> (i128, bool);
+
+    // floats
+    fn __floattisf(i: i128) -> f32;
+    fn __floattidf(i: i128) -> f64;
+    fn __floatuntisf(i: u128) -> f32;
+    fn __floatuntidf(i: u128) -> f64;
+    fn __fixsfti(f: f32) -> i128;
+    fn __fixdfti(f: f64) -> i128;
+    fn __fixunssfti(f: f32) -> u128;
+    fn __fixunsdfti(f: f64) -> u128;
+}
diff --git a/src/constant.rs b/src/constant.rs
index f4cbfb6967f..fcd41c84465 100644
--- a/src/constant.rs
+++ b/src/constant.rs
@@ -85,8 +85,9 @@ pub(crate) fn codegen_tls_ref<'tcx>(
 ) -> CValue<'tcx> {
     let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("tls {:?}", def_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("tls {:?}", def_id));
+    }
     let tls_ptr = fx.bcx.ins().tls_value(fx.pointer_type, local_data_id);
     CValue::by_val(tls_ptr, layout)
 }
@@ -98,8 +99,9 @@ fn codegen_static_ref<'tcx>(
 ) -> CPlace<'tcx> {
     let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("{:?}", def_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("{:?}", def_id));
+    }
     let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
     assert!(!layout.is_unsized(), "unsized statics aren't supported");
     assert!(
@@ -122,7 +124,9 @@ pub(crate) fn codegen_constant<'tcx>(
     };
     let const_val = match const_.val {
         ConstKind::Value(const_val) => const_val,
-        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) if fx.tcx.is_static(def.did) => {
+        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
+            if fx.tcx.is_static(def.did) =>
+        {
             assert!(substs.is_empty());
             assert!(promoted.is_none());
 
@@ -183,8 +187,9 @@ pub(crate) fn codegen_const_value<'tcx>(
                                 data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability);
                             let local_data_id =
                                 fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            #[cfg(debug_assertions)]
-                            fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
+                            if fx.clif_comments.enabled() {
+                                fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
+                            }
                             fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                         }
                         Some(GlobalAlloc::Function(instance)) => {
@@ -199,8 +204,9 @@ pub(crate) fn codegen_const_value<'tcx>(
                             let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
                             let local_data_id =
                                 fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            #[cfg(debug_assertions)]
-                            fx.add_comment(local_data_id, format!("{:?}", def_id));
+                            if fx.clif_comments.enabled() {
+                                fx.add_comment(local_data_id, format!("{:?}", def_id));
+                            }
                             fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                         }
                         None => bug!("missing allocation {:?}", ptr.alloc_id),
@@ -241,8 +247,9 @@ fn pointer_for_allocation<'tcx>(
     let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability);
 
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+    }
     let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
     crate::pointer::Pointer::new(global_ptr)
 }
diff --git a/src/debuginfo/line_info.rs b/src/debuginfo/line_info.rs
index 30ed356c762..8578ab33ced 100644
--- a/src/debuginfo/line_info.rs
+++ b/src/debuginfo/line_info.rs
@@ -39,11 +39,11 @@ fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] {
     #[cfg(unix)]
     {
         use std::os::unix::ffi::OsStrExt;
-        return path.as_bytes();
+        path.as_bytes()
     }
     #[cfg(not(unix))]
     {
-        return path.to_str().unwrap().as_bytes();
+        path.to_str().unwrap().as_bytes()
     }
 }
 
diff --git a/src/driver/aot.rs b/src/driver/aot.rs
index b87dcc41928..ed3bdedddce 100644
--- a/src/driver/aot.rs
+++ b/src/driver/aot.rs
@@ -119,11 +119,10 @@ fn module_codegen(
         tcx.sess.opts.debuginfo != DebugInfo::None,
     );
     super::predefine_mono_items(&mut cx, &mono_items);
-    for (mono_item, (linkage, visibility)) in mono_items {
-        let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+    for (mono_item, _) in mono_items {
         match mono_item {
             MonoItem::Fn(inst) => {
-                cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
+                cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
             }
             MonoItem::Static(def_id) => {
                 crate::constant::codegen_static(&mut cx.constants_cx, def_id)
@@ -163,6 +162,21 @@ pub(super) fn run_aot(
     metadata: EncodedMetadata,
     need_metadata_module: bool,
 ) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
+    use rustc_span::symbol::sym;
+
+    let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
+    let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem);
+    let windows_subsystem = subsystem.map(|subsystem| {
+        if subsystem != sym::windows && subsystem != sym::console {
+            tcx.sess.fatal(&format!(
+                "invalid windows subsystem `{}`, only \
+                                    `windows` and `console` are allowed",
+                subsystem
+            ));
+        }
+        subsystem.to_string()
+    });
+
     let mut work_products = FxHashMap::default();
 
     let cgus = if tcx.sess.opts.output_types.should_codegen() {
@@ -280,7 +294,7 @@ pub(super) fn run_aot(
             allocator_module,
             metadata_module,
             metadata,
-            windows_subsystem: None, // Windows is not yet supported
+            windows_subsystem,
             linker_info: LinkerInfo::new(tcx),
             crate_info: CrateInfo::new(tcx),
         },
diff --git a/src/driver/jit.rs b/src/driver/jit.rs
index 245df03ffb8..dbe1ff083f0 100644
--- a/src/driver/jit.rs
+++ b/src/driver/jit.rs
@@ -5,8 +5,10 @@ use std::cell::RefCell;
 use std::ffi::CString;
 use std::os::raw::{c_char, c_int};
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_codegen_ssa::CrateInfo;
 use rustc_middle::mir::mono::MonoItem;
+use rustc_session::config::EntryFnType;
 
 use cranelift_jit::{JITBuilder, JITModule};
 
@@ -28,20 +30,11 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
     let mut jit_builder =
         JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
     jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
+    crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
     jit_builder.symbols(imported_symbols);
     let mut jit_module = JITModule::new(jit_builder);
     assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
 
-    let sig = Signature {
-        params: vec![
-            AbiParam::new(jit_module.target_config().pointer_type()),
-            AbiParam::new(jit_module.target_config().pointer_type()),
-        ],
-        returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
-        call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
-    };
-    let main_func_id = jit_module.declare_function("main", Linkage::Import, &sig).unwrap();
-
     let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
     let mono_items = cgus
         .iter()
@@ -55,15 +48,12 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
 
     super::time(tcx, "codegen mono items", || {
         super::predefine_mono_items(&mut cx, &mono_items);
-        for (mono_item, (linkage, visibility)) in mono_items {
-            let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+        for (mono_item, _) in mono_items {
             match mono_item {
                 MonoItem::Fn(inst) => match backend_config.codegen_mode {
                     CodegenMode::Aot => unreachable!(),
                     CodegenMode::Jit => {
-                        cx.tcx
-                            .sess
-                            .time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
+                        cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
                     }
                     CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
                 },
@@ -86,24 +76,17 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
         tcx.sess.fatal("Inline asm is not supported in JIT mode");
     }
 
-    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context);
     crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
 
     tcx.sess.abort_if_errors();
 
     jit_module.finalize_definitions();
-
     let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) };
 
-    let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
-
     println!(
         "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
     );
 
-    let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
-        unsafe { ::std::mem::transmute(finalized_main) };
-
     let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
     let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
         .chain(args.split(' '))
@@ -118,12 +101,58 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
     BACKEND_CONFIG.with(|tls_backend_config| {
         assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
     });
-    CURRENT_MODULE
-        .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));
 
-    let ret = f(args.len() as c_int, argv.as_ptr());
+    let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap();
+    let instance = Instance::mono(tcx, main_def_id.to_def_id()).polymorphize(tcx);
 
-    std::process::exit(ret);
+    match entry_ty {
+        EntryFnType::Main => {
+            // FIXME set program arguments somehow
+
+            let main_sig = Signature {
+                params: vec![],
+                returns: vec![],
+                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+            };
+            let main_func_id = jit_module
+                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig)
+                .unwrap();
+            let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
+
+            CURRENT_MODULE.with(|current_module| {
+                assert!(current_module.borrow_mut().replace(jit_module).is_none())
+            });
+
+            let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) };
+            f();
+            std::process::exit(0);
+        }
+        EntryFnType::Start => {
+            let start_sig = Signature {
+                params: vec![
+                    AbiParam::new(jit_module.target_config().pointer_type()),
+                    AbiParam::new(jit_module.target_config().pointer_type()),
+                ],
+                returns: vec![AbiParam::new(
+                    jit_module.target_config().pointer_type(), /*isize*/
+                )],
+                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+            };
+            let start_func_id = jit_module
+                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig)
+                .unwrap();
+            let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
+
+            CURRENT_MODULE.with(|current_module| {
+                assert!(current_module.borrow_mut().replace(jit_module).is_none())
+            });
+
+            let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
+                unsafe { ::std::mem::transmute(finalized_start) };
+            let ret = f(args.len() as c_int, argv.as_ptr());
+            std::process::exit(ret);
+        }
+    }
 }
 
 #[no_mangle]
@@ -144,8 +173,7 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
             jit_module.prepare_for_function_redefine(func_id).unwrap();
 
             let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
-            tcx.sess
-                .time("codegen fn", || crate::base::codegen_fn(&mut cx, instance, Linkage::Export));
+            tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance));
 
             let (global_asm, _debug_context, unwind_context) = cx.finalize();
             assert!(global_asm.is_empty());
@@ -220,7 +248,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
     imported_symbols
 }
 
-pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
+fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
     let tcx = cx.tcx;
 
     let pointer_type = cx.module.target_config().pointer_type();
@@ -267,7 +295,8 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'t
         .define_function(
             func_id,
             &mut Context::for_function(trampoline),
-            &mut cranelift_codegen::binemit::NullTrapSink {},
+            &mut NullTrapSink {},
+            &mut NullStackMapSink {},
         )
         .unwrap();
 }
diff --git a/src/driver/mod.rs b/src/driver/mod.rs
index b994f28ffef..d49182a07b7 100644
--- a/src/driver/mod.rs
+++ b/src/driver/mod.rs
@@ -44,13 +44,19 @@ fn predefine_mono_items<'tcx>(
     mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
 ) {
     cx.tcx.sess.time("predefine functions", || {
+        let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE);
         for &(mono_item, (linkage, visibility)) in mono_items {
             match mono_item {
                 MonoItem::Fn(instance) => {
                     let name = cx.tcx.symbol_name(instance).name.to_string();
                     let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name));
                     let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance);
-                    let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+                    let linkage = crate::linkage::get_clif_linkage(
+                        mono_item,
+                        linkage,
+                        visibility,
+                        is_compiler_builtins,
+                    );
                     cx.module.declare_function(&name, linkage, &sig).unwrap();
                 }
                 MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {}
diff --git a/src/inline_asm.rs b/src/inline_asm.rs
index 5b3df2bd382..1fb5e86aed7 100644
--- a/src/inline_asm.rs
+++ b/src/inline_asm.rs
@@ -20,6 +20,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     if template.is_empty() {
         // Black box
         return;
+    } else if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
+        let true_ = fx.bcx.ins().iconst(types::I32, 1);
+        fx.bcx.ins().trapnz(true_, TrapCode::User(1));
+        return;
     }
 
     let mut slot_size = Size::from_bytes(0);
@@ -193,8 +197,9 @@ fn call_inline_asm<'tcx>(
         offset: None,
         size: u32::try_from(slot_size.bytes()).unwrap(),
     });
-    #[cfg(debug_assertions)]
-    fx.add_comment(stack_slot, "inline asm scratch slot");
+    if fx.clif_comments.enabled() {
+        fx.add_comment(stack_slot, "inline asm scratch slot");
+    }
 
     let inline_asm_func = fx
         .cx
@@ -210,8 +215,9 @@ fn call_inline_asm<'tcx>(
         )
         .unwrap();
     let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(inline_asm_func, asm_name);
+    if fx.clif_comments.enabled() {
+        fx.add_comment(inline_asm_func, asm_name);
+    }
 
     for (_reg, offset, value) in inputs {
         fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 86df71a0dfc..c7ce32b385e 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -88,7 +88,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let idx_bytes = match idx_const {
                     ConstValue::ByRef { alloc, offset } => {
                         let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
-                        let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
+                        let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
                         alloc.get_bytes(fx, ptr, size).unwrap()
                     }
                     _ => unreachable!("{:?}", idx_const),
diff --git a/src/lib.rs b/src/lib.rs
index 8edb883ccb5..720d2a12534 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,13 +1,4 @@
-#![feature(
-    rustc_private,
-    decl_macro,
-    type_alias_impl_trait,
-    associated_type_bounds,
-    never_type,
-    try_blocks,
-    box_patterns,
-    hash_drain_filter
-)]
+#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)]
 #![warn(rust_2018_idioms)]
 #![warn(unused_lifetimes)]
 #![warn(unreachable_pub)]
@@ -57,6 +48,7 @@ mod base;
 mod cast;
 mod codegen_i128;
 mod common;
+mod compiler_builtins;
 mod constant;
 mod debuginfo;
 mod discriminant;
@@ -224,8 +216,10 @@ pub struct CraneliftCodegenBackend {
 
 impl CodegenBackend for CraneliftCodegenBackend {
     fn init(&self, sess: &Session) {
-        if sess.lto() != rustc_session::config::Lto::No && sess.opts.cg.embed_bitcode {
-            sess.warn("LTO is not supported. You may get a linker error.");
+        use rustc_session::config::Lto;
+        match sess.lto() {
+            Lto::No | Lto::ThinLocal => {}
+            Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
         }
     }
 
@@ -240,9 +234,9 @@ impl CodegenBackend for CraneliftCodegenBackend {
         vec![]
     }
 
-    fn codegen_crate<'tcx>(
+    fn codegen_crate(
         &self,
-        tcx: TyCtxt<'tcx>,
+        tcx: TyCtxt<'_>,
         metadata: EncodedMetadata,
         need_metadata_module: bool,
     ) -> Box<dyn Any> {
@@ -252,9 +246,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
             BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
                 .unwrap_or_else(|err| tcx.sess.fatal(&err))
         };
-        let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config);
-
-        res
+        driver::codegen_crate(tcx, metadata, need_metadata_module, config)
     }
 
     fn join_codegen(
@@ -300,9 +292,9 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
     let mut flags_builder = settings::builder();
     flags_builder.enable("is_pic").unwrap();
     flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
-    flags_builder
-        .set("enable_verifier", if cfg!(debug_assertions) { "true" } else { "false" })
-        .unwrap();
+    let enable_verifier =
+        cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok();
+    flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap();
 
     let tls_model = match target_triple.binary_format {
         BinaryFormat::Elf => "elf_gd",
@@ -314,18 +306,17 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
 
     flags_builder.set("enable_simd", "true").unwrap();
 
+    flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
+
     use rustc_session::config::OptLevel;
     match sess.opts.optimize {
         OptLevel::No => {
             flags_builder.set("opt_level", "none").unwrap();
         }
         OptLevel::Less | OptLevel::Default => {}
-        OptLevel::Aggressive => {
+        OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => {
             flags_builder.set("opt_level", "speed_and_size").unwrap();
         }
-        OptLevel::Size | OptLevel::SizeMin => {
-            sess.warn("Optimizing for size is not supported. Just ignoring the request");
-        }
     }
 
     let flags = settings::Flags::new(flags_builder);
diff --git a/src/linkage.rs b/src/linkage.rs
index dc1e2107ce7..a564a59f725 100644
--- a/src/linkage.rs
+++ b/src/linkage.rs
@@ -6,8 +6,10 @@ pub(crate) fn get_clif_linkage(
     mono_item: MonoItem<'_>,
     linkage: RLinkage,
     visibility: Visibility,
+    is_compiler_builtins: bool,
 ) -> Linkage {
     match (linkage, visibility) {
+        (RLinkage::External, Visibility::Default) if is_compiler_builtins => Linkage::Hidden,
         (RLinkage::External, Visibility::Default) => Linkage::Export,
         (RLinkage::Internal, Visibility::Default) => Linkage::Local,
         (RLinkage::External, Visibility::Hidden) => Linkage::Hidden,
diff --git a/src/main_shim.rs b/src/main_shim.rs
index 62e551b186f..a6266f50776 100644
--- a/src/main_shim.rs
+++ b/src/main_shim.rs
@@ -1,3 +1,4 @@
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_hir::LangItem;
 use rustc_session::config::EntryFnType;
 
@@ -100,12 +101,8 @@ pub(crate) fn maybe_create_entry_wrapper(
             bcx.seal_all_blocks();
             bcx.finalize();
         }
-        m.define_function(
-            cmain_func_id,
-            &mut ctx,
-            &mut cranelift_codegen::binemit::NullTrapSink {},
-        )
-        .unwrap();
+        m.define_function(cmain_func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
+            .unwrap();
         unwind_context.add_function(cmain_func_id, &ctx, m.isa());
     }
 }
diff --git a/src/metadata.rs b/src/metadata.rs
index 190c4f45cca..c5189c972cd 100644
--- a/src/metadata.rs
+++ b/src/metadata.rs
@@ -1,11 +1,11 @@
 //! Reading and writing of the rustc metadata for rlibs and dylibs
 
-use std::convert::TryFrom;
 use std::fs::File;
+use std::ops::Deref;
 use std::path::Path;
 
 use rustc_codegen_ssa::METADATA_FILENAME;
-use rustc_data_structures::owning_ref::OwningRef;
+use rustc_data_structures::owning_ref::{OwningRef, StableAddress};
 use rustc_data_structures::rustc_erase_owner;
 use rustc_data_structures::sync::MetadataRef;
 use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader};
@@ -17,38 +17,56 @@ use crate::backend::WriteMetadata;
 
 pub(crate) struct CraneliftMetadataLoader;
 
+struct StableMmap(memmap2::Mmap);
+
+impl Deref for StableMmap {
+    type Target = [u8];
+
+    fn deref(&self) -> &[u8] {
+        &*self.0
+    }
+}
+
+unsafe impl StableAddress for StableMmap {}
+
+fn load_metadata_with(
+    path: &Path,
+    f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
+) -> Result<MetadataRef, String> {
+    let file = File::open(path).map_err(|e| format!("{:?}", e))?;
+    let data = unsafe { memmap2::MmapOptions::new().map_copy_read_only(&file) }
+        .map_err(|e| format!("{:?}", e))?;
+    let metadata = OwningRef::new(StableMmap(data)).try_map(f)?;
+    return Ok(rustc_erase_owner!(metadata.map_owner_box()));
+}
+
 impl MetadataLoader for CraneliftMetadataLoader {
     fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
-        let mut archive = ar::Archive::new(File::open(path).map_err(|e| format!("{:?}", e))?);
-        // Iterate over all entries in the archive:
-        while let Some(entry_result) = archive.next_entry() {
-            let mut entry = entry_result.map_err(|e| format!("{:?}", e))?;
-            if entry.header().identifier() == METADATA_FILENAME.as_bytes() {
-                let mut buf = Vec::with_capacity(
-                    usize::try_from(entry.header().size())
-                        .expect("Rlib metadata file too big to load into memory."),
-                );
-                ::std::io::copy(&mut entry, &mut buf).map_err(|e| format!("{:?}", e))?;
-                let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-                return Ok(rustc_erase_owner!(buf.map_owner_box()));
-            }
-        }
+        load_metadata_with(path, |data| {
+            let archive = object::read::archive::ArchiveFile::parse(&*data)
+                .map_err(|e| format!("{:?}", e))?;
 
-        Err("couldn't find metadata entry".to_string())
+            for entry_result in archive.members() {
+                let entry = entry_result.map_err(|e| format!("{:?}", e))?;
+                if entry.name() == METADATA_FILENAME.as_bytes() {
+                    return Ok(entry.data());
+                }
+            }
+
+            Err("couldn't find metadata entry".to_string())
+        })
     }
 
     fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
         use object::{Object, ObjectSection};
-        let file = std::fs::read(path).map_err(|e| format!("read:{:?}", e))?;
-        let file = object::File::parse(&file).map_err(|e| format!("parse: {:?}", e))?;
-        let buf = file
-            .section_by_name(".rustc")
-            .ok_or("no .rustc section")?
-            .data()
-            .map_err(|e| format!("failed to read .rustc section: {:?}", e))?
-            .to_owned();
-        let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-        Ok(rustc_erase_owner!(buf.map_owner_box()))
+
+        load_metadata_with(path, |data| {
+            let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?;
+            file.section_by_name(".rustc")
+                .ok_or("no .rustc section")?
+                .data()
+                .map_err(|e| format!("failed to read .rustc section: {:?}", e))
+        })
     }
 }
 
diff --git a/src/num.rs b/src/num.rs
index da49e1c6c91..2ebf30da2d8 100644
--- a/src/num.rs
+++ b/src/num.rs
@@ -166,13 +166,11 @@ pub(crate) fn codegen_int_binop<'tcx>(
         BinOp::Shl => {
             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
-            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
             fx.bcx.ins().ishl(lhs, actual_shift)
         }
         BinOp::Shr => {
             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
-            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
             if signed {
                 fx.bcx.ins().sshr(lhs, actual_shift)
             } else {
@@ -387,7 +385,7 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                 let lhs = in_lhs.load_scalar(fx);
                 let rhs = in_rhs.load_scalar(fx);
 
-                return codegen_compare_bin_op(fx, bin_op, false, lhs, rhs);
+                codegen_compare_bin_op(fx, bin_op, false, lhs, rhs)
             }
             BinOp::Offset => {
                 let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty;
@@ -396,10 +394,10 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                 let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
                 let base_val = base.load_scalar(fx);
                 let res = fx.bcx.ins().iadd(base_val, ptr_diff);
-                return CValue::by_val(res, base.layout());
+                CValue::by_val(res, base.layout())
             }
             _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
-        };
+        }
     } else {
         let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx);
         let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx);
diff --git a/src/optimize/stack2reg.rs b/src/optimize/stack2reg.rs
index d111f37f5e4..8bb02a3e558 100644
--- a/src/optimize/stack2reg.rs
+++ b/src/optimize/stack2reg.rs
@@ -181,7 +181,6 @@ impl<'a> OptimizeContext<'a> {
 
 pub(super) fn optimize_function(
     ctx: &mut Context,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))]
     clif_comments: &mut crate::pretty_clif::CommentWriter,
 ) {
     combine_stack_addr_with_load_store(&mut ctx.func);
@@ -192,8 +191,7 @@ pub(super) fn optimize_function(
 
     remove_unused_stack_addr_and_stack_load(&mut opt_ctx);
 
-    #[cfg(debug_assertions)]
-    {
+    if clif_comments.enabled() {
         for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map {
             clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage));
         }
@@ -209,25 +207,27 @@ pub(super) fn optimize_function(
         for load in users.stack_load.clone().into_iter() {
             let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load);
 
-            #[cfg(debug_assertions)]
-            for &store in &potential_stores {
-                clif_comments.add_comment(
-                    load,
-                    format!(
-                        "Potential store -> load forwarding {} -> {} ({:?}, {:?})",
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        opt_ctx.ctx.func.dfg.display_inst(load, None),
-                        spatial_overlap(&opt_ctx.ctx.func, store, load),
-                        temporal_order(&opt_ctx.ctx, store, load),
-                    ),
-                );
+            if clif_comments.enabled() {
+                for &store in &potential_stores {
+                    clif_comments.add_comment(
+                        load,
+                        format!(
+                            "Potential store -> load forwarding {} -> {} ({:?}, {:?})",
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            opt_ctx.ctx.func.dfg.display_inst(load, None),
+                            spatial_overlap(&opt_ctx.ctx.func, store, load),
+                            temporal_order(&opt_ctx.ctx, store, load),
+                        ),
+                    );
+                }
             }
 
             match *potential_stores {
                 [] => {
-                    #[cfg(debug_assertions)]
-                    clif_comments
-                        .add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
+                    if clif_comments.enabled() {
+                        clif_comments
+                            .add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
+                    }
                 }
                 [store]
                     if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full
@@ -237,9 +237,12 @@ pub(super) fn optimize_function(
                     // Only one store could have been the origin of the value.
                     let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0];
 
-                    #[cfg(debug_assertions)]
-                    clif_comments
-                        .add_comment(load, format!("Store to load forward {} -> {}", store, load));
+                    if clif_comments.enabled() {
+                        clif_comments.add_comment(
+                            load,
+                            format!("Store to load forward {} -> {}", store, load),
+                        );
+                    }
 
                     users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value);
                 }
@@ -250,33 +253,35 @@ pub(super) fn optimize_function(
         for store in users.stack_store.clone().into_iter() {
             let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store);
 
-            #[cfg(debug_assertions)]
-            for &load in &potential_loads {
-                clif_comments.add_comment(
-                    store,
-                    format!(
-                        "Potential load from store {} <- {} ({:?}, {:?})",
-                        opt_ctx.ctx.func.dfg.display_inst(load, None),
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        spatial_overlap(&opt_ctx.ctx.func, store, load),
-                        temporal_order(&opt_ctx.ctx, store, load),
-                    ),
-                );
+            if clif_comments.enabled() {
+                for &load in &potential_loads {
+                    clif_comments.add_comment(
+                        store,
+                        format!(
+                            "Potential load from store {} <- {} ({:?}, {:?})",
+                            opt_ctx.ctx.func.dfg.display_inst(load, None),
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            spatial_overlap(&opt_ctx.ctx.func, store, load),
+                            temporal_order(&opt_ctx.ctx, store, load),
+                        ),
+                    );
+                }
             }
 
             if potential_loads.is_empty() {
                 // Never loaded; can safely remove all stores and the stack slot.
                 // FIXME also remove stores when there is always a next store before a load.
 
-                #[cfg(debug_assertions)]
-                clif_comments.add_comment(
-                    store,
-                    format!(
-                        "Remove dead stack store {} of {}",
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        stack_slot.0
-                    ),
-                );
+                if clif_comments.enabled() {
+                    clif_comments.add_comment(
+                        store,
+                        format!(
+                            "Remove dead stack store {} of {}",
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            stack_slot.0
+                        ),
+                    );
+                }
 
                 users.remove_dead_store(&mut opt_ctx.ctx.func, store);
             }
diff --git a/src/pointer.rs b/src/pointer.rs
index 88a78f3214d..31d827f83bf 100644
--- a/src/pointer.rs
+++ b/src/pointer.rs
@@ -39,8 +39,7 @@ impl Pointer {
         Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) }
     }
 
-    #[cfg(debug_assertions)]
-    pub(crate) fn base_and_offset(self) -> (PointerBase, Offset32) {
+    pub(crate) fn debug_base_and_offset(self) -> (PointerBase, Offset32) {
         (self.base, self.offset)
     }
 
diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs
index 9c91b92e515..d22ea3772ee 100644
--- a/src/pretty_clif.rs
+++ b/src/pretty_clif.rs
@@ -69,13 +69,15 @@ use crate::prelude::*;
 
 #[derive(Debug)]
 pub(crate) struct CommentWriter {
+    enabled: bool,
     global_comments: Vec<String>,
     entity_comments: FxHashMap<AnyEntity, String>,
 }
 
 impl CommentWriter {
     pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self {
-        let global_comments = if cfg!(debug_assertions) {
+        let enabled = should_write_ir(tcx);
+        let global_comments = if enabled {
             vec![
                 format!("symbol {}", tcx.symbol_name(instance).name),
                 format!("instance {:?}", instance),
@@ -86,13 +88,17 @@ impl CommentWriter {
             vec![]
         };
 
-        CommentWriter { global_comments, entity_comments: FxHashMap::default() }
+        CommentWriter { enabled, global_comments, entity_comments: FxHashMap::default() }
     }
 }
 
-#[cfg(debug_assertions)]
 impl CommentWriter {
+    pub(crate) fn enabled(&self) -> bool {
+        self.enabled
+    }
+
     pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
+        debug_assert!(self.enabled);
         self.global_comments.push(comment.into());
     }
 
@@ -101,6 +107,8 @@ impl CommentWriter {
         entity: E,
         comment: S,
     ) {
+        debug_assert!(self.enabled);
+
         use std::collections::hash_map::Entry;
         match self.entity_comments.entry(entity.into()) {
             Entry::Occupied(mut occ) => {
@@ -179,7 +187,6 @@ impl FuncWriter for &'_ CommentWriter {
     }
 }
 
-#[cfg(debug_assertions)]
 impl FunctionCx<'_, '_, '_> {
     pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
         self.clif_comments.add_global_comment(comment);
@@ -198,8 +205,8 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
     tcx.sess.opts.output_types.contains_key(&OutputType::LlvmAssembly)
 }
 
-pub(crate) fn write_ir_file<'tcx>(
-    tcx: TyCtxt<'tcx>,
+pub(crate) fn write_ir_file(
+    tcx: TyCtxt<'_>,
     name: &str,
     write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
 ) {
@@ -217,10 +224,7 @@ pub(crate) fn write_ir_file<'tcx>(
 
     let clif_file_name = clif_output_dir.join(name);
 
-    let res: std::io::Result<()> = try {
-        let mut file = std::fs::File::create(clif_file_name)?;
-        write(&mut file)?;
-    };
+    let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
     if let Err(err) = res {
         tcx.sess.warn(&format!("error writing ir file: {}", err));
     }
diff --git a/src/trap.rs b/src/trap.rs
index bb63d72addf..1ab0703e981 100644
--- a/src/trap.rs
+++ b/src/trap.rs
@@ -17,8 +17,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
         )
         .unwrap();
     let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    {
+    if fx.clif_comments.enabled() {
         fx.add_comment(puts, "puts");
     }
 
diff --git a/src/value_and_place.rs b/src/value_and_place.rs
index cffaf79ded1..b97d3900984 100644
--- a/src/value_and_place.rs
+++ b/src/value_and_place.rs
@@ -2,7 +2,6 @@
 
 use crate::prelude::*;
 
-use cranelift_codegen::entity::EntityRef;
 use cranelift_codegen::ir::immediates::Offset32;
 
 fn codegen_field<'tcx>(
@@ -414,7 +413,7 @@ impl<'tcx> CPlace<'tcx> {
         self,
         fx: &mut FunctionCx<'_, '_, 'tcx>,
         from: CValue<'tcx>,
-        #[cfg_attr(not(debug_assertions), allow(unused_variables))] method: &'static str,
+        method: &'static str,
     ) {
         fn transmute_value<'tcx>(
             fx: &mut FunctionCx<'_, '_, 'tcx>,
@@ -462,8 +461,7 @@ impl<'tcx> CPlace<'tcx> {
 
         assert_eq!(self.layout().size, from.layout().size);
 
-        #[cfg(debug_assertions)]
-        {
+        if fx.clif_comments.enabled() {
             use cranelift_codegen::cursor::{Cursor, CursorPosition};
             let cur_block = match fx.bcx.cursor().position() {
                 CursorPosition::After(block) => block,
@@ -707,6 +705,19 @@ pub(crate) fn assert_assignable<'tcx>(
             }
             // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed
         }
+        (&ty::Adt(adt_def_a, substs_a), &ty::Adt(adt_def_b, substs_b))
+            if adt_def_a.did == adt_def_b.did =>
+        {
+            let mut types_a = substs_a.types();
+            let mut types_b = substs_b.types();
+            loop {
+                match (types_a.next(), types_b.next()) {
+                    (Some(a), Some(b)) => assert_assignable(fx, a, b),
+                    (None, None) => return,
+                    (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+                }
+            }
+        }
         _ => {
             assert_eq!(
                 from_ty, to_ty,