3e24a39357
This PR optimizes 32-bit unsigned division by constants when the magic
constant is 33 bits (IsAdd=true case in UnsignedDivisionByConstantInfo)
on 64-bit targets.
## Overview
Compiler optimization for constant division of `uint32_t` variables
(such as `x / 7`) is based on the method
proposed by Granlund and Montgomery in 1994 (hereafter referred to as
the GM method).
However, the GM method for the IsAdd=true case was optimized for 32-bit
CPUs, not 64-bit CPUs.
This patch provides optimizations specifically for 64-bit CPUs (such as
x86_64 and Apple M-series).
A simple benchmark demonstrates over 60% speedup on both Intel Xeon and
Apple M4 processors.
## The GM Method
The GM method for `x / 7` can be expressed in C code as follows,
where the constants `c` and `a` are magic numbers determined by the
divisor:
```cpp
uint32_t udiv_original(uint32_t x) {
uint64_t v = x * c;
v >>= 32;
uint32_t t = uint32_t(x) - uint32_t(v);
t >>= 1;
t += uint32_t(v);
t >>= a - 33;
return t;
}
```
For example, division by 7 on x86_64 generates 7 instructions:
```asm
movl %edi, %eax
imulq $613566757, %rax, %rax
shrq $32, %rax
subl %eax, %edi
shrl %edi
addl %edi, %eax
shrl $2, %eax
```
## Proposed Solution
This patch generates the following optimized code:
```cpp
uint32_t udiv_optimized(uint32_t x) {
uint128_t v = uint128_t(x) * ((c + 0x100000000) << (64 - a));
return uint32_t(v >> 64);
}
```
Since a 64-bit right shift of a 128-bit variable extracts the upper 64
bits,
this code eliminates the need for shifts after multiplication.
The implementation pre-shifts the 33-bit magic constant `c = 2^32 +
Magic` left by `(64-a)` bits
and uses the high 64 bits of a 64 x 64 -> 128 bit multiplication
directly.
This eliminates the add/sub/shift sequence.
After optimization, division by 7 becomes 4 instructions (or 3 with
BMI2):
```asm
# Standard (4 instructions)
movl %edi, %eax
movabsq $2635249153617166336, %rcx
mulq %rcx
movq %rdx, %rax
# With BMI2 (3 instructions)
movl %edi, %edx
movabsq $2635249153617166336, %rax
mulxq %rax, %rax, %rax
```
62 lines
1.9 KiB
LLVM
62 lines
1.9 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc < %s -mtriple=aarch64-unknown-linux | FileCheck %s
|
|
|
|
; Test optimization of 32-bit unsigned division by constants with 33-bit magic
|
|
; constants (IsAdd=true) on AArch64. The optimization uses the umulh instruction.
|
|
|
|
define i32 @udiv_by_7(i32 %x) nounwind {
|
|
; CHECK-LABEL: udiv_by_7:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: mov x8, #2684354560 // =0xa0000000
|
|
; CHECK-NEXT: mov w9, w0
|
|
; CHECK-NEXT: movk x8, #18724, lsl #32
|
|
; CHECK-NEXT: movk x8, #9362, lsl #48
|
|
; CHECK-NEXT: umulh x0, x9, x8
|
|
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; CHECK-NEXT: ret
|
|
%div = udiv i32 %x, 7
|
|
ret i32 %div
|
|
}
|
|
|
|
define i32 @udiv_by_19(i32 %x) nounwind {
|
|
; CHECK-LABEL: udiv_by_19:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: mov x8, #1476395008 // =0x58000000
|
|
; CHECK-NEXT: mov w9, w0
|
|
; CHECK-NEXT: movk x8, #17246, lsl #32
|
|
; CHECK-NEXT: movk x8, #3449, lsl #48
|
|
; CHECK-NEXT: umulh x0, x9, x8
|
|
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; CHECK-NEXT: ret
|
|
%div = udiv i32 %x, 19
|
|
ret i32 %div
|
|
}
|
|
|
|
define i32 @udiv_by_21(i32 %x) nounwind {
|
|
; CHECK-LABEL: udiv_by_21:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: mov x8, #939524096 // =0x38000000
|
|
; CHECK-NEXT: mov w9, w0
|
|
; CHECK-NEXT: movk x8, #49932, lsl #32
|
|
; CHECK-NEXT: movk x8, #3120, lsl #48
|
|
; CHECK-NEXT: umulh x0, x9, x8
|
|
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; CHECK-NEXT: ret
|
|
%div = udiv i32 %x, 21
|
|
ret i32 %div
|
|
}
|
|
|
|
; Test non-optimized case
|
|
define i32 @udiv_by_3(i32 %x) nounwind {
|
|
; CHECK-LABEL: udiv_by_3:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: mov w8, #43691 // =0xaaab
|
|
; CHECK-NEXT: movk w8, #43690, lsl #16
|
|
; CHECK-NEXT: umull x8, w0, w8
|
|
; CHECK-NEXT: lsr x0, x8, #33
|
|
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; CHECK-NEXT: ret
|
|
%div = udiv i32 %x, 3
|
|
ret i32 %div
|
|
}
|