Rollup merge of rust-lang#72632 - jonas-schievink:dest-prop, r=oli-obk

Implement a generic Destination Propagation optimization on MIR

This takes the work that was originally started by @eddyb in rust-lang#47954, and then explored by me in rust-lang#71003, and implements it in a general (ie. not limited to acyclic CFGs) and dataflow-driven way (so that no additional infrastructure in rustc is needed).

The pass is configured to run at `mir-opt-level=2` and higher only. To enable it by default, some followup work on it is still needed:
* Performance needs to be evaluated. I did some light optimization work and tested against `tuple-stress`, which caused trouble in my last attempt, but didn't go much in depth here.
  * We can also enable the pass only at `opt-level=2` and higher, if it is too slow to run in debug mode, but fine when optimizations run anyways.
* Debuginfo needs to be fixed after locals are merged. I did not look into what is required for this.
* Live ranges of locals (aka `StorageLive` and `StorageDead`) are currently deleted. We either need to decide that this is fine, or if not, merge the variable's live ranges (or remove these statements entirely – rust-lang#68622).

Some benchmarks of the pass were done in rust-lang#72635.

compiler/rustc_mir/src/dataflow/impls/mod.rs

@@ -204,7 +204,7 @@ impl<'a, 'tcx> HasMoveData<'tcx> for DefinitelyInitializedPlaces<'a, 'tcx> {
 
 /// `EverInitializedPlaces` tracks all places that might have ever been
 /// initialized upon reaching a particular point in the control flow
-/// for a function, without an intervening `Storage Dead`.
+/// for a function, without an intervening `StorageDead`.
 ///
 /// This dataflow is used to determine if an immutable local variable may
 /// be assigned to.

compiler/rustc_mir/src/lib.rs

@@ -6,6 +6,7 @@ Rust MIR: a lowered representation of Rust.
 
 #![feature(nll)]
 #![feature(in_band_lifetimes)]
+#![feature(bindings_after_at)]
 #![feature(bool_to_option)]
 #![feature(box_patterns)]
 #![feature(box_syntax)]

compiler/rustc_mir/src/transform/mod.rs

@@ -24,6 +24,7 @@ pub mod cleanup_post_borrowck;
 pub mod const_prop;
 pub mod copy_prop;
 pub mod deaggregator;
+pub mod dest_prop;
 pub mod dump_mir;
 pub mod elaborate_drops;
 pub mod generator;
@@ -467,6 +468,7 @@ fn run_optimization_passes<'tcx>(
         &simplify_comparison_integral::SimplifyComparisonIntegral,
         &simplify_try::SimplifyArmIdentity,
         &simplify_try::SimplifyBranchSame,
+        &dest_prop::DestinationPropagation,
         &copy_prop::CopyPropagation,
         &simplify_branches::SimplifyBranches::new("after-copy-prop"),
         &remove_noop_landing_pads::RemoveNoopLandingPads,

compiler/rustc_mir/src/transform/nrvo.rs

@@ -36,6 +36,12 @@ impl<'tcx> MirPass<'tcx> for RenameReturnPlace {
             return;
         }
 
+        if tcx.sess.opts.debugging_opts.mir_opt_level >= 2 {
+            // The `DestinationPropagation` pass runs at level 2, so this pass is redundant (and
+            // fails some asserts).
+            return;
+        }
+
         let returned_local = match local_eligible_for_nrvo(body) {
             Some(l) => l,
             None => {

src/test/mir-opt/const_allocation.main.ConstProp.after.32bit.mir

@@ -24,42 +24,42 @@ fn main() -> () {
 }
 
 alloc0 (static: FOO, size: 8, align: 4) {
-    ╾─alloc17─╼ 03 00 00 00                         │ ╾──╼....
+    ╾─alloc14─╼ 03 00 00 00                         │ ╾──╼....
 }
 
-alloc17 (size: 48, align: 4) {
+alloc14 (size: 48, align: 4) {
     0x00 │ 00 00 00 00 __ __ __ __ ╾─alloc4──╼ 00 00 00 00 │ ....░░░░╾──╼....
-    0x10 │ 00 00 00 00 __ __ __ __ ╾─alloc8──╼ 02 00 00 00 │ ....░░░░╾──╼....
-    0x20 │ 01 00 00 00 2a 00 00 00 ╾─alloc13─╼ 03 00 00 00 │ ....*...╾──╼....
+    0x10 │ 00 00 00 00 __ __ __ __ ╾─alloc7──╼ 02 00 00 00 │ ....░░░░╾──╼....
+    0x20 │ 01 00 00 00 2a 00 00 00 ╾─alloc11─╼ 03 00 00 00 │ ....*...╾──╼....
 }
 
 alloc4 (size: 0, align: 4) {}
 
-alloc8 (size: 16, align: 4) {
-    ╾─alloc7──╼ 03 00 00 00 ╾─alloc9──╼ 03 00 00 00 │ ╾──╼....╾──╼....
+alloc7 (size: 16, align: 4) {
+    ╾─alloc6──╼ 03 00 00 00 ╾─alloc8──╼ 03 00 00 00 │ ╾──╼....╾──╼....
 }
 
-alloc7 (size: 3, align: 1) {
+alloc6 (size: 3, align: 1) {
     66 6f 6f                                        │ foo
 }
 
-alloc9 (size: 3, align: 1) {
+alloc8 (size: 3, align: 1) {
     62 61 72                                        │ bar
 }
 
-alloc13 (size: 24, align: 4) {
-    0x00 │ ╾─alloc12─╼ 03 00 00 00 ╾─alloc14─╼ 03 00 00 00 │ ╾──╼....╾──╼....
-    0x10 │ ╾─alloc15─╼ 04 00 00 00                         │ ╾──╼....
+alloc11 (size: 24, align: 4) {
+    0x00 │ ╾─alloc10─╼ 03 00 00 00 ╾─alloc12─╼ 03 00 00 00 │ ╾──╼....╾──╼....
+    0x10 │ ╾─alloc13─╼ 04 00 00 00                         │ ╾──╼....
 }
 
-alloc12 (size: 3, align: 1) {
+alloc10 (size: 3, align: 1) {
     6d 65 68                                        │ meh
 }
 
-alloc14 (size: 3, align: 1) {
+alloc12 (size: 3, align: 1) {
     6d 6f 70                                        │ mop
 }
 
-alloc15 (size: 4, align: 1) {
+alloc13 (size: 4, align: 1) {
     6d c3 b6 70                                     │ m..p
 }

src/test/mir-opt/const_allocation.main.ConstProp.after.64bit.mir

@@ -24,46 +24,46 @@ fn main() -> () {
 }
 
 alloc0 (static: FOO, size: 16, align: 8) {
-    ╾───────alloc17───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
+    ╾───────alloc14───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
 }
 
-alloc17 (size: 72, align: 8) {
+alloc14 (size: 72, align: 8) {
     0x00 │ 00 00 00 00 __ __ __ __ ╾───────alloc4────────╼ │ ....░░░░╾──────╼
     0x10 │ 00 00 00 00 00 00 00 00 00 00 00 00 __ __ __ __ │ ............░░░░
-    0x20 │ ╾───────alloc8────────╼ 02 00 00 00 00 00 00 00 │ ╾──────╼........
-    0x30 │ 01 00 00 00 2a 00 00 00 ╾───────alloc13───────╼ │ ....*...╾──────╼
+    0x20 │ ╾───────alloc7────────╼ 02 00 00 00 00 00 00 00 │ ╾──────╼........
+    0x30 │ 01 00 00 00 2a 00 00 00 ╾───────alloc11───────╼ │ ....*...╾──────╼
     0x40 │ 03 00 00 00 00 00 00 00                         │ ........
 }
 
 alloc4 (size: 0, align: 8) {}
 
-alloc8 (size: 32, align: 8) {
-    0x00 │ ╾───────alloc7────────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
-    0x10 │ ╾───────alloc9────────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
+alloc7 (size: 32, align: 8) {
+    0x00 │ ╾───────alloc6────────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
+    0x10 │ ╾───────alloc8────────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
 }
 
-alloc7 (size: 3, align: 1) {
+alloc6 (size: 3, align: 1) {
     66 6f 6f                                        │ foo
 }
 
-alloc9 (size: 3, align: 1) {
+alloc8 (size: 3, align: 1) {
     62 61 72                                        │ bar
 }
 
-alloc13 (size: 48, align: 8) {
-    0x00 │ ╾───────alloc12───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
-    0x10 │ ╾───────alloc14───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
-    0x20 │ ╾───────alloc15───────╼ 04 00 00 00 00 00 00 00 │ ╾──────╼........
+alloc11 (size: 48, align: 8) {
+    0x00 │ ╾───────alloc10───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
+    0x10 │ ╾───────alloc12───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
+    0x20 │ ╾───────alloc13───────╼ 04 00 00 00 00 00 00 00 │ ╾──────╼........
 }
 
-alloc12 (size: 3, align: 1) {
+alloc10 (size: 3, align: 1) {
     6d 65 68                                        │ meh
 }
 
-alloc14 (size: 3, align: 1) {
+alloc12 (size: 3, align: 1) {
     6d 6f 70                                        │ mop
 }
 
-alloc15 (size: 4, align: 1) {
+alloc13 (size: 4, align: 1) {
     6d c3 b6 70                                     │ m..p
 }

src/test/mir-opt/const_allocation2.main.ConstProp.after.32bit.mir

@@ -24,41 +24,41 @@ fn main() -> () {
 }
 
 alloc0 (static: FOO, size: 8, align: 4) {
-    ╾─alloc23─╼ 03 00 00 00                         │ ╾──╼....
+    ╾─alloc20─╼ 03 00 00 00                         │ ╾──╼....
 }
 
-alloc23 (size: 48, align: 4) {
+alloc20 (size: 48, align: 4) {
     0x00 │ 00 00 00 00 __ __ __ __ ╾─alloc8──╼ 00 00 00 00 │ ....░░░░╾──╼....
-    0x10 │ 00 00 00 00 __ __ __ __ ╾─alloc13─╼ 02 00 00 00 │ ....░░░░╾──╼....
-    0x20 │ 01 00 00 00 2a 00 00 00 ╾─alloc21─╼ 03 00 00 00 │ ....*...╾──╼....
+    0x10 │ 00 00 00 00 __ __ __ __ ╾─alloc12─╼ 02 00 00 00 │ ....░░░░╾──╼....
+    0x20 │ 01 00 00 00 2a 00 00 00 ╾─alloc19─╼ 03 00 00 00 │ ....*...╾──╼....
 }
 
 alloc8 (size: 0, align: 4) {}
 
-alloc13 (size: 8, align: 4) {
-    ╾─alloc11─╼ ╾─alloc12─╼                         │ ╾──╼╾──╼
+alloc12 (size: 8, align: 4) {
+    ╾─alloc10─╼ ╾─alloc11─╼                         │ ╾──╼╾──╼
 }
 
-alloc11 (size: 1, align: 1) {
+alloc10 (size: 1, align: 1) {
     05                                              │ .
 }
 
-alloc12 (size: 1, align: 1) {
+alloc11 (size: 1, align: 1) {
     06                                              │ .
 }
 
-alloc21 (size: 12, align: 4) {
-    ╾─a17+0x3─╼ ╾─alloc18─╼ ╾─a20+0x2─╼             │ ╾──╼╾──╼╾──╼
+alloc19 (size: 12, align: 4) {
+    ╾─a15+0x3─╼ ╾─alloc16─╼ ╾─a18+0x2─╼             │ ╾──╼╾──╼╾──╼
 }
 
-alloc17 (size: 4, align: 1) {
+alloc15 (size: 4, align: 1) {
     2a 45 15 6f                                     │ *E.o
 }
 
-alloc18 (size: 1, align: 1) {
+alloc16 (size: 1, align: 1) {
     2a                                              │ *
 }
 
-alloc20 (size: 4, align: 1) {
+alloc18 (size: 4, align: 1) {
     2a 45 15 6f                                     │ *E.o
 }

src/test/mir-opt/const_allocation2.main.ConstProp.after.64bit.mir

@@ -24,44 +24,44 @@ fn main() -> () {
 }
 
 alloc0 (static: FOO, size: 16, align: 8) {
-    ╾───────alloc23───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
+    ╾───────alloc20───────╼ 03 00 00 00 00 00 00 00 │ ╾──────╼........
 }
 
-alloc23 (size: 72, align: 8) {
+alloc20 (size: 72, align: 8) {
     0x00 │ 00 00 00 00 __ __ __ __ ╾───────alloc8────────╼ │ ....░░░░╾──────╼
     0x10 │ 00 00 00 00 00 00 00 00 00 00 00 00 __ __ __ __ │ ............░░░░
-    0x20 │ ╾───────alloc13───────╼ 02 00 00 00 00 00 00 00 │ ╾──────╼........
-    0x30 │ 01 00 00 00 2a 00 00 00 ╾───────alloc21───────╼ │ ....*...╾──────╼
+    0x20 │ ╾───────alloc12───────╼ 02 00 00 00 00 00 00 00 │ ╾──────╼........
+    0x30 │ 01 00 00 00 2a 00 00 00 ╾───────alloc19───────╼ │ ....*...╾──────╼
     0x40 │ 03 00 00 00 00 00 00 00                         │ ........
 }
 
 alloc8 (size: 0, align: 8) {}
 
-alloc13 (size: 16, align: 8) {
-    ╾───────alloc11───────╼ ╾───────alloc12───────╼ │ ╾──────╼╾──────╼
+alloc12 (size: 16, align: 8) {
+    ╾───────alloc10───────╼ ╾───────alloc11───────╼ │ ╾──────╼╾──────╼
 }
 
-alloc11 (size: 1, align: 1) {
+alloc10 (size: 1, align: 1) {
     05                                              │ .
 }
 
-alloc12 (size: 1, align: 1) {
+alloc11 (size: 1, align: 1) {
     06                                              │ .
 }
 
-alloc21 (size: 24, align: 8) {
-    0x00 │ ╾─────alloc17+0x3─────╼ ╾───────alloc18───────╼ │ ╾──────╼╾──────╼
-    0x10 │ ╾─────alloc20+0x2─────╼                         │ ╾──────╼
+alloc19 (size: 24, align: 8) {
+    0x00 │ ╾─────alloc15+0x3─────╼ ╾───────alloc16───────╼ │ ╾──────╼╾──────╼
+    0x10 │ ╾─────alloc18+0x2─────╼                         │ ╾──────╼
 }
 
-alloc17 (size: 4, align: 1) {
+alloc15 (size: 4, align: 1) {
     2a 45 15 6f                                     │ *E.o
 }
 
-alloc18 (size: 1, align: 1) {
+alloc16 (size: 1, align: 1) {
     2a                                              │ *
 }
 
-alloc20 (size: 4, align: 1) {
+alloc18 (size: 4, align: 1) {
     2a 45 15 6f                                     │ *E.o
 }

src/test/mir-opt/const_allocation3.main.ConstProp.after.32bit.mir

@@ -24,10 +24,10 @@ fn main() -> () {
 }
 
 alloc0 (static: FOO, size: 4, align: 4) {
-    ╾─alloc9──╼                                     │ ╾──╼
+    ╾─alloc3──╼                                     │ ╾──╼
 }
 
-alloc9 (size: 168, align: 1) {
+alloc3 (size: 168, align: 1) {
     0x00 │ ab ab ab ab ab ab ab ab ab ab ab ab ab ab ab ab │ ................
     0x10 │ ab ab ab ab ab ab ab ab ab ab ab ab ╾─alloc4──╼ │ ............╾──╼
     0x20 │ 01 ef cd ab 00 00 00 00 00 00 00 00 00 00 00 00 │ ................

src/test/mir-opt/const_allocation3.main.ConstProp.after.64bit.mir

@@ -24,10 +24,10 @@ fn main() -> () {
 }
 
 alloc0 (static: FOO, size: 8, align: 8) {
-    ╾───────alloc9────────╼                         │ ╾──────╼
+    ╾───────alloc3────────╼                         │ ╾──────╼
 }
 
-alloc9 (size: 180, align: 1) {
+alloc3 (size: 180, align: 1) {
     0x00 │ ab ab ab ab ab ab ab ab ab ab ab ab ab ab ab ab │ ................
     0x10 │ ab ab ab ab ab ab ab ab ab ab ab ab ╾──alloc4── │ ............╾───
     0x20 │ ──────────╼ 01 ef cd ab 00 00 00 00 00 00 00 00 │ ───╼............

src/test/mir-opt/copy_propagation.test.CopyPropagation.diff

@@ -6,19 +6,14 @@
       let mut _0: u32;                     // return place in scope 0 at $DIR/copy_propagation.rs:3:20: 3:23
       let _2: u32;                         // in scope 0 at $DIR/copy_propagation.rs:4:9: 4:10
       scope 1 {
--         debug y => _2;                   // in scope 1 at $DIR/copy_propagation.rs:4:9: 4:10
-+         debug y => _1;                   // in scope 1 at $DIR/copy_propagation.rs:4:9: 4:10
+          debug y => _0;                   // in scope 1 at $DIR/copy_propagation.rs:4:9: 4:10
       }
 
       bb0: {
--         StorageLive(_2);                 // scope 0 at $DIR/copy_propagation.rs:4:9: 4:10
--         _2 = _1;                         // scope 0 at $DIR/copy_propagation.rs:4:13: 4:14
--         _0 = _2;                         // scope 1 at $DIR/copy_propagation.rs:5:5: 5:6
--         StorageDead(_2);                 // scope 0 at $DIR/copy_propagation.rs:6:1: 6:2
-+         nop;                             // scope 0 at $DIR/copy_propagation.rs:4:9: 4:10
-+         nop;                             // scope 0 at $DIR/copy_propagation.rs:4:13: 4:14
-+         _0 = _1;                         // scope 1 at $DIR/copy_propagation.rs:5:5: 5:6
-+         nop;                             // scope 0 at $DIR/copy_propagation.rs:6:1: 6:2
+          nop;                             // scope 0 at $DIR/copy_propagation.rs:4:9: 4:10
+          _0 = _1;                         // scope 0 at $DIR/copy_propagation.rs:4:13: 4:14
+          nop;                             // scope 1 at $DIR/copy_propagation.rs:5:5: 5:6
+          nop;                             // scope 0 at $DIR/copy_propagation.rs:6:1: 6:2
           return;                          // scope 0 at $DIR/copy_propagation.rs:6:2: 6:2
       }
   }

src/test/mir-opt/copy_propagation_arg.arg_src.CopyPropagation.diff

@@ -6,15 +6,15 @@
       let mut _0: i32;                     // return place in scope 0 at $DIR/copy_propagation_arg.rs:27:27: 27:30
       let _2: i32;                         // in scope 0 at $DIR/copy_propagation_arg.rs:28:9: 28:10
       scope 1 {
-          debug y => _2;                   // in scope 1 at $DIR/copy_propagation_arg.rs:28:9: 28:10
+          debug y => _0;                   // in scope 1 at $DIR/copy_propagation_arg.rs:28:9: 28:10
       }
 
       bb0: {
-          StorageLive(_2);                 // scope 0 at $DIR/copy_propagation_arg.rs:28:9: 28:10
-          _2 = _1;                         // scope 0 at $DIR/copy_propagation_arg.rs:28:13: 28:14
+          nop;                             // scope 0 at $DIR/copy_propagation_arg.rs:28:9: 28:10
+          _0 = _1;                         // scope 0 at $DIR/copy_propagation_arg.rs:28:13: 28:14
           _1 = const 123_i32;              // scope 1 at $DIR/copy_propagation_arg.rs:29:5: 29:12
-          _0 = _2;                         // scope 1 at $DIR/copy_propagation_arg.rs:30:5: 30:6
-          StorageDead(_2);                 // scope 0 at $DIR/copy_propagation_arg.rs:31:1: 31:2
+          nop;                             // scope 1 at $DIR/copy_propagation_arg.rs:30:5: 30:6
+          nop;                             // scope 0 at $DIR/copy_propagation_arg.rs:31:1: 31:2
           return;                          // scope 0 at $DIR/copy_propagation_arg.rs:31:2: 31:2
       }
   }

src/test/mir-opt/copy_propagation_arg.baz.CopyPropagation.diff

@@ -7,10 +7,10 @@
       let mut _2: i32;                     // in scope 0 at $DIR/copy_propagation_arg.rs:23:9: 23:10
 
       bb0: {
-          StorageLive(_2);                 // scope 0 at $DIR/copy_propagation_arg.rs:23:9: 23:10
-          _2 = _1;                         // scope 0 at $DIR/copy_propagation_arg.rs:23:9: 23:10
-          _1 = move _2;                    // scope 0 at $DIR/copy_propagation_arg.rs:23:5: 23:10
-          StorageDead(_2);                 // scope 0 at $DIR/copy_propagation_arg.rs:23:9: 23:10
+          nop;                             // scope 0 at $DIR/copy_propagation_arg.rs:23:9: 23:10
+          nop;                             // scope 0 at $DIR/copy_propagation_arg.rs:23:9: 23:10
+          nop;                             // scope 0 at $DIR/copy_propagation_arg.rs:23:5: 23:10
+          nop;                             // scope 0 at $DIR/copy_propagation_arg.rs:23:9: 23:10
           _0 = const ();                   // scope 0 at $DIR/copy_propagation_arg.rs:21:20: 24:2
           return;                          // scope 0 at $DIR/copy_propagation_arg.rs:24:2: 24:2
       }

src/test/mir-opt/copy_propagation_arg.foo.CopyPropagation.diff

@@ -8,19 +8,19 @@
       let mut _3: u8;                      // in scope 0 at $DIR/copy_propagation_arg.rs:11:15: 11:16
 
       bb0: {
-          StorageLive(_2);                 // scope 0 at $DIR/copy_propagation_arg.rs:11:9: 11:17
+          nop;                             // scope 0 at $DIR/copy_propagation_arg.rs:11:9: 11:17
           StorageLive(_3);                 // scope 0 at $DIR/copy_propagation_arg.rs:11:15: 11:16
           _3 = _1;                         // scope 0 at $DIR/copy_propagation_arg.rs:11:15: 11:16
-          _2 = dummy(move _3) -> bb1;      // scope 0 at $DIR/copy_propagation_arg.rs:11:9: 11:17
+          _1 = dummy(move _3) -> bb1;      // scope 0 at $DIR/copy_propagation_arg.rs:11:9: 11:17
                                            // mir::Constant
                                            // + span: $DIR/copy_propagation_arg.rs:11:9: 11:14
                                            // + literal: Const { ty: fn(u8) -> u8 {dummy}, val: Value(Scalar(<ZST>)) }
       }
 
       bb1: {
           StorageDead(_3);                 // scope 0 at $DIR/copy_propagation_arg.rs:11:16: 11:17
-          _1 = move _2;                    // scope 0 at $DIR/copy_propagation_arg.rs:11:5: 11:17
-          StorageDead(_2);                 // scope 0 at $DIR/copy_propagation_arg.rs:11:16: 11:17
+          nop;                             // scope 0 at $DIR/copy_propagation_arg.rs:11:5: 11:17
+          nop;                             // scope 0 at $DIR/copy_propagation_arg.rs:11:16: 11:17
           _0 = const ();                   // scope 0 at $DIR/copy_propagation_arg.rs:9:19: 12:2
           return;                          // scope 0 at $DIR/copy_propagation_arg.rs:12:2: 12:2
       }