3e24a39357
This PR optimizes 32-bit unsigned division by constants when the magic
constant is 33 bits (IsAdd=true case in UnsignedDivisionByConstantInfo)
on 64-bit targets.
## Overview
Compiler optimization for constant division of `uint32_t` variables
(such as `x / 7`) is based on the method
proposed by Granlund and Montgomery in 1994 (hereafter referred to as
the GM method).
However, the GM method for the IsAdd=true case was optimized for 32-bit
CPUs, not 64-bit CPUs.
This patch provides optimizations specifically for 64-bit CPUs (such as
x86_64 and Apple M-series).
A simple benchmark demonstrates over 60% speedup on both Intel Xeon and
Apple M4 processors.
## The GM Method
The GM method for `x / 7` can be expressed in C code as follows,
where the constants `c` and `a` are magic numbers determined by the
divisor:
```cpp
uint32_t udiv_original(uint32_t x) {
uint64_t v = x * c;
v >>= 32;
uint32_t t = uint32_t(x) - uint32_t(v);
t >>= 1;
t += uint32_t(v);
t >>= a - 33;
return t;
}
```
For example, division by 7 on x86_64 generates 7 instructions:
```asm
movl %edi, %eax
imulq $613566757, %rax, %rax
shrq $32, %rax
subl %eax, %edi
shrl %edi
addl %edi, %eax
shrl $2, %eax
```
## Proposed Solution
This patch generates the following optimized code:
```cpp
uint32_t udiv_optimized(uint32_t x) {
uint128_t v = uint128_t(x) * ((c + 0x100000000) << (64 - a));
return uint32_t(v >> 64);
}
```
Since a 64-bit right shift of a 128-bit variable extracts the upper 64
bits,
this code eliminates the need for shifts after multiplication.
The implementation pre-shifts the 33-bit magic constant `c = 2^32 +
Magic` left by `(64-a)` bits
and uses the high 64 bits of a 64 x 64 -> 128 bit multiplication
directly.
This eliminates the add/sub/shift sequence.
After optimization, division by 7 becomes 4 instructions (or 3 with
BMI2):
```asm
# Standard (4 instructions)
movl %edi, %eax
movabsq $2635249153617166336, %rcx
mulq %rcx
movq %rdx, %rax
# With BMI2 (3 instructions)
movl %edi, %edx
movabsq $2635249153617166336, %rax
mulxq %rax, %rax, %rax
```
142 lines
4.6 KiB
LLVM
142 lines
4.6 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefix=X64
|
|
; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+bmi2 | FileCheck %s --check-prefix=X64-BMI2
|
|
; RUN: llc < %s -mtriple=i386-unknown-linux-gnu | FileCheck %s --check-prefix=X86
|
|
|
|
; Test optimization of 32-bit unsigned division by constants with 33-bit magic
|
|
; constants (IsAdd=true) on 64-bit targets. The optimization uses pre-shifted
|
|
; constants and 64x64->128 bit multiplication to reduce instruction count.
|
|
|
|
define i32 @udiv_by_7(i32 %x) nounwind {
|
|
; X64-LABEL: udiv_by_7:
|
|
; X64: # %bb.0:
|
|
; X64-NEXT: movl %edi, %eax
|
|
; X64-NEXT: movabsq $2635249153617166336, %rcx # imm = 0x24924924A0000000
|
|
; X64-NEXT: mulq %rcx
|
|
; X64-NEXT: movq %rdx, %rax
|
|
; X64-NEXT: # kill: def $eax killed $eax killed $rax
|
|
; X64-NEXT: retq
|
|
;
|
|
; X64-BMI2-LABEL: udiv_by_7:
|
|
; X64-BMI2: # %bb.0:
|
|
; X64-BMI2-NEXT: movl %edi, %edx
|
|
; X64-BMI2-NEXT: movabsq $2635249153617166336, %rax # imm = 0x24924924A0000000
|
|
; X64-BMI2-NEXT: mulxq %rax, %rax, %rax
|
|
; X64-BMI2-NEXT: # kill: def $eax killed $eax killed $rax
|
|
; X64-BMI2-NEXT: retq
|
|
;
|
|
; X86-LABEL: udiv_by_7:
|
|
; X86: # %bb.0:
|
|
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
|
; X86-NEXT: movl $613566757, %edx # imm = 0x24924925
|
|
; X86-NEXT: movl %ecx, %eax
|
|
; X86-NEXT: mull %edx
|
|
; X86-NEXT: subl %edx, %ecx
|
|
; X86-NEXT: shrl %ecx
|
|
; X86-NEXT: addl %edx, %ecx
|
|
; X86-NEXT: shrl $2, %ecx
|
|
; X86-NEXT: movl %ecx, %eax
|
|
; X86-NEXT: retl
|
|
%div = udiv i32 %x, 7
|
|
ret i32 %div
|
|
}
|
|
|
|
define i32 @udiv_by_19(i32 %x) nounwind {
|
|
; X64-LABEL: udiv_by_19:
|
|
; X64: # %bb.0:
|
|
; X64-NEXT: movl %edi, %eax
|
|
; X64-NEXT: movabsq $970881267157434368, %rcx # imm = 0xD79435E58000000
|
|
; X64-NEXT: mulq %rcx
|
|
; X64-NEXT: movq %rdx, %rax
|
|
; X64-NEXT: # kill: def $eax killed $eax killed $rax
|
|
; X64-NEXT: retq
|
|
;
|
|
; X64-BMI2-LABEL: udiv_by_19:
|
|
; X64-BMI2: # %bb.0:
|
|
; X64-BMI2-NEXT: movl %edi, %edx
|
|
; X64-BMI2-NEXT: movabsq $970881267157434368, %rax # imm = 0xD79435E58000000
|
|
; X64-BMI2-NEXT: mulxq %rax, %rax, %rax
|
|
; X64-BMI2-NEXT: # kill: def $eax killed $eax killed $rax
|
|
; X64-BMI2-NEXT: retq
|
|
;
|
|
; X86-LABEL: udiv_by_19:
|
|
; X86: # %bb.0:
|
|
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
|
; X86-NEXT: movl ${{-?[0-9]+}}, %edx # imm = 0xAF286BCB
|
|
; X86-NEXT: movl %ecx, %eax
|
|
; X86-NEXT: mull %edx
|
|
; X86-NEXT: subl %edx, %ecx
|
|
; X86-NEXT: shrl %ecx
|
|
; X86-NEXT: addl %edx, %ecx
|
|
; X86-NEXT: shrl $4, %ecx
|
|
; X86-NEXT: movl %ecx, %eax
|
|
; X86-NEXT: retl
|
|
%div = udiv i32 %x, 19
|
|
ret i32 %div
|
|
}
|
|
|
|
define i32 @udiv_by_21(i32 %x) nounwind {
|
|
; X64-LABEL: udiv_by_21:
|
|
; X64: # %bb.0:
|
|
; X64-NEXT: movl %edi, %eax
|
|
; X64-NEXT: movabsq $878416384583794688, %rcx # imm = 0xC30C30C38000000
|
|
; X64-NEXT: mulq %rcx
|
|
; X64-NEXT: movq %rdx, %rax
|
|
; X64-NEXT: # kill: def $eax killed $eax killed $rax
|
|
; X64-NEXT: retq
|
|
;
|
|
; X64-BMI2-LABEL: udiv_by_21:
|
|
; X64-BMI2: # %bb.0:
|
|
; X64-BMI2-NEXT: movl %edi, %edx
|
|
; X64-BMI2-NEXT: movabsq $878416384583794688, %rax # imm = 0xC30C30C38000000
|
|
; X64-BMI2-NEXT: mulxq %rax, %rax, %rax
|
|
; X64-BMI2-NEXT: # kill: def $eax killed $eax killed $rax
|
|
; X64-BMI2-NEXT: retq
|
|
;
|
|
; X86-LABEL: udiv_by_21:
|
|
; X86: # %bb.0:
|
|
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
|
|
; X86-NEXT: movl ${{-?[0-9]+}}, %edx # imm = 0x86186187
|
|
; X86-NEXT: movl %ecx, %eax
|
|
; X86-NEXT: mull %edx
|
|
; X86-NEXT: subl %edx, %ecx
|
|
; X86-NEXT: shrl %ecx
|
|
; X86-NEXT: addl %edx, %ecx
|
|
; X86-NEXT: shrl $4, %ecx
|
|
; X86-NEXT: movl %ecx, %eax
|
|
; X86-NEXT: retl
|
|
%div = udiv i32 %x, 21
|
|
ret i32 %div
|
|
}
|
|
|
|
; Test non-optimized case: divisor that doesn't use IsAdd
|
|
define i32 @udiv_by_3(i32 %x) nounwind {
|
|
; X64-LABEL: udiv_by_3:
|
|
; X64: # %bb.0:
|
|
; X64-NEXT: movl %edi, %ecx
|
|
; X64-NEXT: movl $2863311531, %eax # imm = 0xAAAAAAAB
|
|
; X64-NEXT: imulq %rcx, %rax
|
|
; X64-NEXT: shrq $33, %rax
|
|
; X64-NEXT: # kill: def $eax killed $eax killed $rax
|
|
; X64-NEXT: retq
|
|
;
|
|
; X64-BMI2-LABEL: udiv_by_3:
|
|
; X64-BMI2: # %bb.0:
|
|
; X64-BMI2-NEXT: movl %edi, %ecx
|
|
; X64-BMI2-NEXT: movl $2863311531, %eax # imm = 0xAAAAAAAB
|
|
; X64-BMI2-NEXT: imulq %rcx, %rax
|
|
; X64-BMI2-NEXT: shrq $33, %rax
|
|
; X64-BMI2-NEXT: # kill: def $eax killed $eax killed $rax
|
|
; X64-BMI2-NEXT: retq
|
|
;
|
|
; X86-LABEL: udiv_by_3:
|
|
; X86: # %bb.0:
|
|
; X86-NEXT: movl ${{-?[0-9]+}}, %eax # imm = 0xAAAAAAAB
|
|
; X86-NEXT: mull {{[0-9]+}}(%esp)
|
|
; X86-NEXT: movl %edx, %eax
|
|
; X86-NEXT: shrl %eax
|
|
; X86-NEXT: retl
|
|
%div = udiv i32 %x, 3
|
|
ret i32 %div
|
|
}
|