Skip to content

Commit 4051305

Browse files
borsbors
committed
Auto merge of #112238 - scottmcm:mir-add-unchecked, r=cjgillot
Promote unchecked integer math to MIR `BinOp`s So slice indexing by a range gets down to one basic block, for example. r? cjgillot
2 parents 6895110 + c780e55 commit 4051305

File tree

40 files changed

+544
-356
lines changed

40 files changed

+544
-356
lines changed

‎compiler/rustc_codegen_cranelift/src/codegen_i128.rs

+5-4
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ pub(crate) fn maybe_codegen<'tcx>(
2222

2323
match bin_op {
2424
BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => None,
25-
BinOp::Add | BinOp::Sub => None,
26-
BinOp::Mul => {
25+
BinOp::Add | BinOp::AddUnchecked | BinOp::Sub | BinOp::SubUnchecked => None,
26+
BinOp::Mul | BinOp::MulUnchecked => {
2727
let args = [lhs.load_scalar(fx), rhs.load_scalar(fx)];
2828
let ret_val = fx.lib_call(
2929
"__multi3",
@@ -69,7 +69,7 @@ pub(crate) fn maybe_codegen<'tcx>(
6969
}
7070
}
7171
BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => None,
72-
BinOp::Shl | BinOp::Shr => None,
72+
BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => None,
7373
}
7474
}
7575

@@ -131,9 +131,10 @@ pub(crate) fn maybe_codegen_checked<'tcx>(
131131
fx.lib_call(name, param_types, vec![], &args);
132132
Some(out_place.to_cvalue(fx))
133133
}
134+
BinOp::AddUnchecked | BinOp::SubUnchecked | BinOp::MulUnchecked => unreachable!(),
134135
BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
135136
BinOp::Div | BinOp::Rem => unreachable!(),
136137
BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => unreachable!(),
137-
BinOp::Shl | BinOp::Shr => unreachable!(),
138+
BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => unreachable!(),
138139
}
139140
}

‎compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

+3-17
Original file line numberDiff line numberDiff line change
@@ -472,25 +472,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
472472
ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
473473
}
474474

475-
sym::unchecked_add
476-
| sym::unchecked_sub
477-
| sym::unchecked_mul
478-
| sym::exact_div
479-
| sym::unchecked_shl
480-
| sym::unchecked_shr => {
475+
sym::exact_div => {
481476
intrinsic_args!(fx, args => (x, y); intrinsic);
482477

483-
// FIXME trap on overflow
484-
let bin_op = match intrinsic {
485-
sym::unchecked_add => BinOp::Add,
486-
sym::unchecked_sub => BinOp::Sub,
487-
sym::unchecked_mul => BinOp::Mul,
488-
sym::exact_div => BinOp::Div,
489-
sym::unchecked_shl => BinOp::Shl,
490-
sym::unchecked_shr => BinOp::Shr,
491-
_ => unreachable!(),
492-
};
493-
let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
478+
// FIXME trap on inexact
479+
let res = crate::num::codegen_int_binop(fx, BinOp::Div, x, y);
494480
ret.write_cvalue(fx, res);
495481
}
496482
sym::saturating_add | sym::saturating_sub => {

‎compiler/rustc_codegen_cranelift/src/num.rs

+10-6
Original file line numberDiff line numberDiff line change
@@ -128,10 +128,11 @@ pub(crate) fn codegen_int_binop<'tcx>(
128128
let rhs = in_rhs.load_scalar(fx);
129129

130130
let b = fx.bcx.ins();
131+
// FIXME trap on overflow for the Unchecked versions
131132
let val = match bin_op {
132-
BinOp::Add => b.iadd(lhs, rhs),
133-
BinOp::Sub => b.isub(lhs, rhs),
134-
BinOp::Mul => b.imul(lhs, rhs),
133+
BinOp::Add | BinOp::AddUnchecked => b.iadd(lhs, rhs),
134+
BinOp::Sub | BinOp::SubUnchecked => b.isub(lhs, rhs),
135+
BinOp::Mul | BinOp::MulUnchecked => b.imul(lhs, rhs),
135136
BinOp::Div => {
136137
if signed {
137138
b.sdiv(lhs, rhs)
@@ -149,16 +150,19 @@ pub(crate) fn codegen_int_binop<'tcx>(
149150
BinOp::BitXor => b.bxor(lhs, rhs),
150151
BinOp::BitAnd => b.band(lhs, rhs),
151152
BinOp::BitOr => b.bor(lhs, rhs),
152-
BinOp::Shl => b.ishl(lhs, rhs),
153-
BinOp::Shr => {
153+
BinOp::Shl | BinOp::ShlUnchecked => b.ishl(lhs, rhs),
154+
BinOp::Shr | BinOp::ShrUnchecked => {
154155
if signed {
155156
b.sshr(lhs, rhs)
156157
} else {
157158
b.ushr(lhs, rhs)
158159
}
159160
}
161+
BinOp::Offset => unreachable!("Offset is not an integer operation"),
160162
// Compare binops handles by `codegen_binop`.
161-
_ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
163+
BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge => {
164+
unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty);
165+
}
162166
};
163167

164168
CValue::by_val(val, in_lhs.layout())

‎compiler/rustc_codegen_ssa/src/common.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ pub fn build_langcall<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
132132
// all shifts). For 32- and 64-bit types, this matches the semantics
133133
// of Java. (See related discussion on #1877 and #10183.)
134134

135-
pub fn build_unchecked_lshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
135+
pub fn build_masked_lshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
136136
bx: &mut Bx,
137137
lhs: Bx::Value,
138138
rhs: Bx::Value,
@@ -143,7 +143,7 @@ pub fn build_unchecked_lshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
143143
bx.shl(lhs, rhs)
144144
}
145145

146-
pub fn build_unchecked_rshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
146+
pub fn build_masked_rshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
147147
bx: &mut Bx,
148148
lhs_t: Ty<'tcx>,
149149
lhs: Bx::Value,

‎compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

+6-43
Original file line numberDiff line numberDiff line change
@@ -211,52 +211,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
211211
args[1].val.unaligned_volatile_store(bx, dst);
212212
return;
213213
}
214-
| sym::unchecked_shl
215-
| sym::unchecked_shr
216-
| sym::unchecked_add
217-
| sym::unchecked_sub
218-
| sym::unchecked_mul
219-
| sym::exact_div => {
214+
sym::exact_div => {
220215
let ty = arg_tys[0];
221216
match int_type_width_signed(ty, bx.tcx()) {
222-
Some((_width, signed)) => match name {
223-
sym::exact_div => {
224-
if signed {
225-
bx.exactsdiv(args[0].immediate(), args[1].immediate())
226-
} else {
227-
bx.exactudiv(args[0].immediate(), args[1].immediate())
228-
}
229-
}
230-
sym::unchecked_shl => bx.shl(args[0].immediate(), args[1].immediate()),
231-
sym::unchecked_shr => {
232-
if signed {
233-
bx.ashr(args[0].immediate(), args[1].immediate())
234-
} else {
235-
bx.lshr(args[0].immediate(), args[1].immediate())
236-
}
237-
}
238-
sym::unchecked_add => {
239-
if signed {
240-
bx.unchecked_sadd(args[0].immediate(), args[1].immediate())
241-
} else {
242-
bx.unchecked_uadd(args[0].immediate(), args[1].immediate())
243-
}
244-
}
245-
sym::unchecked_sub => {
246-
if signed {
247-
bx.unchecked_ssub(args[0].immediate(), args[1].immediate())
248-
} else {
249-
bx.unchecked_usub(args[0].immediate(), args[1].immediate())
250-
}
251-
}
252-
sym::unchecked_mul => {
253-
if signed {
254-
bx.unchecked_smul(args[0].immediate(), args[1].immediate())
255-
} else {
256-
bx.unchecked_umul(args[0].immediate(), args[1].immediate())
257-
}
217+
Some((_width, signed)) => {
218+
if signed {
219+
bx.exactsdiv(args[0].immediate(), args[1].immediate())
220+
} else {
221+
bx.exactudiv(args[0].immediate(), args[1].immediate())
258222
}
259-
_ => bug!(),
260223
},
261224
None => {
262225
bx.tcx().sess.emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

+31-2
Original file line numberDiff line numberDiff line change
@@ -798,20 +798,41 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
798798
bx.add(lhs, rhs)
799799
}
800800
}
801+
mir::BinOp::AddUnchecked => {
802+
if is_signed {
803+
bx.unchecked_sadd(lhs, rhs)
804+
} else {
805+
bx.unchecked_uadd(lhs, rhs)
806+
}
807+
}
801808
mir::BinOp::Sub => {
802809
if is_float {
803810
bx.fsub(lhs, rhs)
804811
} else {
805812
bx.sub(lhs, rhs)
806813
}
807814
}
815+
mir::BinOp::SubUnchecked => {
816+
if is_signed {
817+
bx.unchecked_ssub(lhs, rhs)
818+
} else {
819+
bx.unchecked_usub(lhs, rhs)
820+
}
821+
}
808822
mir::BinOp::Mul => {
809823
if is_float {
810824
bx.fmul(lhs, rhs)
811825
} else {
812826
bx.mul(lhs, rhs)
813827
}
814828
}
829+
mir::BinOp::MulUnchecked => {
830+
if is_signed {
831+
bx.unchecked_smul(lhs, rhs)
832+
} else {
833+
bx.unchecked_umul(lhs, rhs)
834+
}
835+
}
815836
mir::BinOp::Div => {
816837
if is_float {
817838
bx.fdiv(lhs, rhs)
@@ -848,8 +869,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
848869
bx.inbounds_gep(llty, lhs, &[rhs])
849870
}
850871
}
851-
mir::BinOp::Shl => common::build_unchecked_lshift(bx, lhs, rhs),
852-
mir::BinOp::Shr => common::build_unchecked_rshift(bx, input_ty, lhs, rhs),
872+
mir::BinOp::Shl => common::build_masked_lshift(bx, lhs, rhs),
873+
mir::BinOp::ShlUnchecked => {
874+
let rhs = base::cast_shift_expr_rhs(bx, lhs, rhs);
875+
bx.shl(lhs, rhs)
876+
}
877+
mir::BinOp::Shr => common::build_masked_rshift(bx, input_ty, lhs, rhs),
878+
mir::BinOp::ShrUnchecked => {
879+
let rhs = base::cast_shift_expr_rhs(bx, lhs, rhs);
880+
if is_signed { bx.ashr(lhs, rhs) } else { bx.lshr(lhs, rhs) }
881+
}
853882
mir::BinOp::Ne
854883
| mir::BinOp::Lt
855884
| mir::BinOp::Gt

‎compiler/rustc_const_eval/src/interpret/intrinsics.rs

-31
Original file line numberDiff line numberDiff line change
@@ -234,37 +234,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
234234
let r = self.read_immediate(&args[1])?;
235235
self.exact_div(&l, &r, dest)?;
236236
}
237-
sym::unchecked_shl
238-
| sym::unchecked_shr
239-
| sym::unchecked_add
240-
| sym::unchecked_sub
241-
| sym::unchecked_mul => {
242-
let l = self.read_immediate(&args[0])?;
243-
let r = self.read_immediate(&args[1])?;
244-
let bin_op = match intrinsic_name {
245-
sym::unchecked_shl => BinOp::Shl,
246-
sym::unchecked_shr => BinOp::Shr,
247-
sym::unchecked_add => BinOp::Add,
248-
sym::unchecked_sub => BinOp::Sub,
249-
sym::unchecked_mul => BinOp::Mul,
250-
_ => bug!(),
251-
};
252-
let (val, overflowed, _ty) = self.overflowing_binary_op(bin_op, &l, &r)?;
253-
if overflowed {
254-
let layout = self.layout_of(substs.type_at(0))?;
255-
let r_val = r.to_scalar().to_bits(layout.size)?;
256-
if let sym::unchecked_shl | sym::unchecked_shr = intrinsic_name {
257-
throw_ub_custom!(
258-
fluent::const_eval_overflow_shift,
259-
val = r_val,
260-
name = intrinsic_name
261-
);
262-
} else {
263-
throw_ub_custom!(fluent::const_eval_overflow, name = intrinsic_name);
264-
}
265-
}
266-
self.write_scalar(val, dest)?;
267-
}
268237
sym::rotate_left | sym::rotate_right => {
269238
// rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
270239
// rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))

0 commit comments

Comments
 (0)