Skip to content

Commit

Permalink
s390x: Use reg_addr register class
Browse files Browse the repository at this point in the history 
Refs: rust-lang/rust#119431
  • Loading branch information
@taiki-e
taiki-e committed Jan 6, 2024
1 parent 2b9b7f9 commit 9ee7f57
Show file tree
Hide file tree
Showing 5 changed files with 473 additions and 31 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,8 @@ jobs:
target: riscv64gc-unknown-linux-gnu
- rust: nightly
target: s390x-unknown-linux-gnu
- rust: nightly-2024-01-04 # Rust 1.77, LLVM 17 (newest version that does not support reg_addr register class)
target: s390x-unknown-linux-gnu
- rust: nightly-2023-05-09 # Rust 1.71, LLVM 16 (oldest version we can implement atomic RMWs on this target)
target: s390x-unknown-linux-gnu
- rust: '1.59'
Expand Down
4 changes: 4 additions & 0 deletions build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ fn main() {
&& version.probe(71, 2023, 5, 8)
&& is_allowed_feature("asm_experimental_arch")
{
// https://github.com/rust-lang/rust/pull/119431 merged in Rust 1.77 (nightly-2024-01-05).
if !version.probe(77, 2024, 1, 4) {
println!("cargo:rustc-cfg=atomic_maybe_uninit_s390x_no_reg_addr");
}
println!("cargo:rustc-cfg=atomic_maybe_uninit_unstable_asm_experimental_arch");
}
}
Expand Down
1 change: 1 addition & 0 deletions src/arch/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ mod powerpc;
mod riscv;
#[cfg(target_arch = "s390x")]
#[cfg(atomic_maybe_uninit_unstable_asm_experimental_arch)]
#[cfg_attr(atomic_maybe_uninit_s390x_no_reg_addr, path = "s390x_no_reg_addr.rs")]
mod s390x;
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
mod x86;
60 changes: 29 additions & 31 deletions src/arch/s390x.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,8 @@ macro_rules! atomic_load_store {
asm!(
// (atomic) load from src to out
concat!("l", $l_suffix, " {out}, 0({src})"),
src = in(reg) ptr_reg!(src),
src = in(reg_addr) ptr_reg!(src),
out = lateout(reg) out,
out("r0") _,
options(nostack, preserves_flags),
);
}
Expand All @@ -79,9 +78,8 @@ macro_rules! atomic_load_store {
// (atomic) store val to dst
concat!("st", $asm_suffix, " {val}, 0({dst})"),
$fence,
dst = in(reg) ptr_reg!(dst),
dst = in(reg_addr) ptr_reg!(dst),
val = in(reg) val,
out("r0") _,
options(nostack, preserves_flags),
)
};
Expand Down Expand Up @@ -126,13 +124,13 @@ macro_rules! atomic {
// atomic swap is always SeqCst.
asm!(
// (atomic) swap (CAS loop)
concat!("l", $asm_suffix, " %r0, 0({dst})"),
concat!("l", $asm_suffix, " {out}, 0({dst})"),
"2:",
concat!("cs", $asm_suffix, " %r0, {val}, 0({dst})"),
concat!("cs", $asm_suffix, " {out}, {val}, 0({dst})"),
"jl 2b",
dst = in(reg) ptr_reg!(dst),
dst = in(reg_addr) ptr_reg!(dst),
val = in(reg) val,
out("r0") out,
out = out(reg) out,
// Do not use `preserves_flags` because CS modifies the condition code.
options(nostack),
);
Expand All @@ -158,13 +156,13 @@ macro_rules! atomic {
// compare_exchange is always SeqCst.
asm!(
// (atomic) CAS
concat!("cs", $asm_suffix, " %r0, {new}, 0({dst})"),
concat!("cs", $asm_suffix, " {old}, {new}, 0({dst})"),
// store condition code
"ipm {r}",
dst = in(reg) ptr_reg!(dst),
dst = in(reg_addr) ptr_reg!(dst),
new = in(reg) new,
r = lateout(reg) r,
inout("r0") old => out,
old = inout(reg) old => out,
// Do not use `preserves_flags` because CS modifies the condition code.
options(nostack),
);
Expand Down Expand Up @@ -195,21 +193,21 @@ macro_rules! atomic_sub_word {
// Based on assemblies generated by rustc/LLVM.
// See also create_sub_word_mask_values.
asm!(
"l %r0, 0({dst})",
"l {prev}, 0({dst})",
"2:",
"rll {tmp}, %r0, 0({shift})",
"rll {tmp}, {prev}, 0({shift})",
concat!("risbg {tmp}, {val}, 32, ", $risbg_swap),
"rll {tmp}, {tmp}, 0({shift_c})",
"cs %r0, {tmp}, 0({dst})",
"cs {prev}, {tmp}, 0({dst})",
"jl 2b",
concat!("rll {out}, %r0, ", $bits ,"({shift})"),
dst = in(reg) ptr_reg!(dst),
concat!("rll {out}, {prev}, ", $bits ,"({shift})"),
dst = in(reg_addr) ptr_reg!(dst),
val = in(reg) val,
out = lateout(reg) out,
shift = in(reg) shift as u32,
shift_c = in(reg) complement(shift as u32),
shift = in(reg_addr) shift as u32,
shift_c = in(reg_addr) complement(shift as u32),
tmp = out(reg) _,
out("r0") _, // prev
prev = out(reg) _,
// Do not use `preserves_flags` because CS modifies the condition code.
options(nostack),
);
Expand Down Expand Up @@ -239,26 +237,26 @@ macro_rules! atomic_sub_word {
asm!(
"l {prev}, 0({dst})",
"2:",
concat!("rll %r0, {prev}, ", $bits ,"({shift})"),
concat!("risbg {new}, %r0, 32, ", $risbg_cas, ", 0"),
concat!("ll", $asm_suffix, "r %r0, %r0"),
"cr %r0, {old}",
concat!("rll {out}, {prev}, ", $bits ,"({shift})"),
concat!("risbg {new}, {out}, 32, ", $risbg_cas, ", 0"),
concat!("ll", $asm_suffix, "r {out}, {out}"),
"cr {out}, {old}",
"jlh 3f",
concat!("rll {tmp}, {new}, -", $bits ,"({shift_c})"),
"cs {prev}, {tmp}, 0({dst})",
"jl 2b",
"3:",
// store condition code
"ipm {r}",
dst = in(reg) ptr_reg!(dst),
dst = in(reg_addr) ptr_reg!(dst),
prev = out(reg) _,
old = in(reg) crate::utils::zero_extend(old),
new = inout(reg) new => _,
shift = in(reg) shift as u32,
shift_c = in(reg) complement(shift as u32),
shift = in(reg_addr) shift as u32,
shift_c = in(reg_addr) complement(shift as u32),
tmp = out(reg) _,
r = lateout(reg) r,
out("r0") out,
out = out(reg) out,
// Do not use `preserves_flags` because CS modifies the condition code.
options(nostack),
);
Expand Down Expand Up @@ -298,7 +296,7 @@ macro_rules! atomic128 {
asm!(
// (atomic) load from src to out pair
"lpq %r0, 0({src})",
src = in(reg) ptr_reg!(src),
src = in(reg_addr) ptr_reg!(src),
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
out("r0") prev_hi,
out("r1") prev_lo,
Expand Down Expand Up @@ -326,7 +324,7 @@ macro_rules! atomic128 {
// (atomic) store val pair to dst
"stpq %r0, 0({dst})",
$fence,
dst = in(reg) ptr_reg!(dst),
dst = in(reg_addr) ptr_reg!(dst),
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
in("r0") val.pair.hi,
in("r1") val.pair.lo,
Expand Down Expand Up @@ -373,7 +371,7 @@ macro_rules! atomic128 {
"2:",
"cdsg %r0, %r12, 0({dst})",
"jl 2b",
dst = inout(reg) ptr_reg!(dst) => _,
dst = in(reg_addr) ptr_reg!(dst),
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
out("r0") prev_hi,
out("r1") prev_lo,
Expand Down Expand Up @@ -409,7 +407,7 @@ macro_rules! atomic128 {
"cdsg %r0, %r12, 0({dst})",
// store condition code
"ipm {r}",
dst = in(reg) ptr_reg!(dst),
dst = in(reg_addr) ptr_reg!(dst),
r = lateout(reg) r,
// Quadword atomic instructions work with even/odd pair of specified register and subsequent register.
inout("r0") old.pair.hi => prev_hi,
Expand Down
Loading

0 comments on commit 9ee7f57

Please sign in to comment.