From ee3c19fd7fb0d2fa8e2f7e49bd0bb7c1394fcda9 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 29 Mar 2018 22:03:05 +0000 Subject: [PATCH] [X86] Add ReadAfterLds to some 3 src instructions Sometimes the operand comes after the memory operand so we need 5 ReadDefaults first. I suspect we also need to do something for the mask operand for masked avx512 instructions? I'm not sure if the mask should be ReadAfterLd or not since it can mask faults. If it shouldn't be ReadAfterLd then we're probably wrong for zero masking instructions already. Differential Revision: https://reviews.llvm.org/D44726 llvm-svn: 328834 --- llvm/lib/Target/X86/X86InstrFMA.td | 52 +++++++++++++++------- llvm/lib/Target/X86/X86InstrSSE.td | 7 ++- llvm/lib/Target/X86/X86InstrShiftRotate.td | 2 +- llvm/lib/Target/X86/X86InstrXOP.td | 26 ++++++++--- 4 files changed, 64 insertions(+), 23 deletions(-) diff --git a/llvm/lib/Target/X86/X86InstrFMA.td b/llvm/lib/Target/X86/X86InstrFMA.td index 35fa45590fc6..3928f0f01937 100644 --- a/llvm/lib/Target/X86/X86InstrFMA.td +++ b/llvm/lib/Target/X86/X86InstrFMA.td @@ -51,7 +51,7 @@ multiclass fma3p_rm_213 opc, string OpcodeStr, RegisterClass RC, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, (VT (Op RC:$src2, RC:$src1, (MemFrag addr:$src3))))]>, - Sched<[WriteFMALd, ReadAfterLd]>; + Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>; } multiclass fma3p_rm_231 opc, string OpcodeStr, RegisterClass RC, @@ -70,7 +70,8 @@ multiclass fma3p_rm_231 opc, string OpcodeStr, RegisterClass RC, !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, (VT (Op RC:$src2, (MemFrag addr:$src3), - RC:$src1)))]>, Sched<[WriteFMALd, ReadAfterLd]>; + RC:$src1)))]>, + Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>; } multiclass fma3p_rm_132 opc, string OpcodeStr, RegisterClass RC, @@ -91,7 +92,8 @@ multiclass fma3p_rm_132 opc, string OpcodeStr, RegisterClass RC, !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, (VT (Op (MemFrag addr:$src3), RC:$src1, - RC:$src2)))]>, Sched<[WriteFMALd, ReadAfterLd]>; + RC:$src2)))]>, + Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>; } let Constraints = "$src1 = $dst", hasSideEffects = 0, isCommutable = 1 in @@ -184,7 +186,7 @@ multiclass fma3s_rm_213 opc, string OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, (OpNode RC:$src2, RC:$src1, (load addr:$src3)))]>, - Sched<[WriteFMALd, ReadAfterLd]>; + Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>; } multiclass fma3s_rm_231 opc, string OpcodeStr, @@ -204,7 +206,7 @@ multiclass fma3s_rm_231 opc, string OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, (OpNode RC:$src2, (load addr:$src3), RC:$src1))]>, - Sched<[WriteFMALd, ReadAfterLd]>; + Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>; } multiclass fma3s_rm_132 opc, string OpcodeStr, @@ -226,7 +228,7 @@ multiclass fma3s_rm_132 opc, string OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), [(set RC:$dst, (OpNode (load addr:$src3), RC:$src1, RC:$src2))]>, - Sched<[WriteFMALd, ReadAfterLd]>; + Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>; } let Constraints = "$src1 = $dst", isCommutable = 1, hasSideEffects = 0 in @@ -270,7 +272,7 @@ multiclass fma3s_rm_int opc, string OpcodeStr, (ins RC:$src1, RC:$src2, memopr:$src3), !strconcat(OpcodeStr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), - []>, Sched<[WriteFMALd, ReadAfterLd]>; + []>, Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>; } // The FMA 213 form is created for lowering of scalar FMA intrinscis @@ -374,14 +376,19 @@ multiclass fma4s opc, string OpcodeStr, RegisterClass RC, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set RC:$dst, (OpNode RC:$src1, RC:$src2, (mem_frag addr:$src3)))]>, VEX_W, VEX_LIG, - Sched<[WriteFMALd, ReadAfterLd]>; + Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>; def mr : FMA4S, VEX_LIG, - Sched<[WriteFMALd, ReadAfterLd]>; + Sched<[WriteFMALd, ReadAfterLd, + // x86memop:$src2 + ReadDefault, ReadDefault, ReadDefault, ReadDefault, + ReadDefault, + // RC:$src3 + ReadAfterLd]>; // For disassembler let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in def rr_REV : FMA4S, VEX_W, VEX_LIG, - Sched<[WriteFMALd, ReadAfterLd]>; + Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>; def mr_Int : FMA4S_Int, - VEX_LIG, Sched<[WriteFMALd, ReadAfterLd]>; + VEX_LIG, Sched<[WriteFMALd, ReadAfterLd, + // memop:$src2 + ReadDefault, ReadDefault, ReadDefault, + ReadDefault, ReadDefault, + // VR128::$src3 + ReadAfterLd]>; let hasSideEffects = 0 in def rr_Int_REV : FMA4S_Int opc, string OpcodeStr, SDNode OpNode, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set VR128:$dst, (OpNode VR128:$src1, VR128:$src2, (ld_frag128 addr:$src3)))]>, VEX_W, - Sched<[WriteFMALd, ReadAfterLd]>; + Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>; def mr : FMA4, - Sched<[WriteFMALd, ReadAfterLd]>; + Sched<[WriteFMALd, ReadAfterLd, + // f128mem:$src2 + ReadDefault, ReadDefault, ReadDefault, ReadDefault, + ReadDefault, + // VR128::$src3 + ReadAfterLd]>; let isCommutable = 1 in def Yrr : FMA4 opc, string OpcodeStr, SDNode OpNode, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set VR256:$dst, (OpNode VR256:$src1, VR256:$src2, (ld_frag256 addr:$src3)))]>, VEX_W, VEX_L, - Sched<[WriteFMALd, ReadAfterLd]>; + Sched<[WriteFMALd, ReadAfterLd, ReadAfterLd]>; def Ymr : FMA4, VEX_L, - Sched<[WriteFMALd, ReadAfterLd]>; + Sched<[WriteFMALd, ReadAfterLd, + // f256mem:$src2 + ReadDefault, ReadDefault, ReadDefault, ReadDefault, + ReadDefault, + // VR256::$src3 + ReadAfterLd]>; // For disassembler let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in { def rr_REV : FMA4 opc, string OpcodeStr, (IntId RC:$src1, (bitconvert (mem_frag addr:$src2)), RC:$src3))], itins.rm, SSEPackedInt>, TAPD, VEX_4V, - Sched<[itins.Sched.Folded, ReadAfterLd]>; + Sched<[itins.Sched.Folded, ReadAfterLd, + // x86memop:$src2 + ReadDefault, ReadDefault, ReadDefault, ReadDefault, + ReadDefault, + // RC::$src3 + ReadAfterLd]>; } let Predicates = [HasAVX] in { diff --git a/llvm/lib/Target/X86/X86InstrShiftRotate.td b/llvm/lib/Target/X86/X86InstrShiftRotate.td index 43e1752f2df2..ef7367e0bb51 100644 --- a/llvm/lib/Target/X86/X86InstrShiftRotate.td +++ b/llvm/lib/Target/X86/X86InstrShiftRotate.td @@ -897,7 +897,7 @@ let hasSideEffects = 0 in { // x86memop:$src1 ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, - // RC:$src1 + // RC:$src2 ReadAfterLd]>; } } diff --git a/llvm/lib/Target/X86/X86InstrXOP.td b/llvm/lib/Target/X86/X86InstrXOP.td index fc0cd5beab25..0aaaeebefa23 100644 --- a/llvm/lib/Target/X86/X86InstrXOP.td +++ b/llvm/lib/Target/X86/X86InstrXOP.td @@ -287,7 +287,7 @@ multiclass xop4op opc, string OpcodeStr, SDNode OpNode, [(set VR128:$dst, (vt128 (OpNode (vt128 VR128:$src1), (vt128 VR128:$src2), (vt128 (bitconvert (loadv2i64 addr:$src3))))))]>, - XOP_4V, VEX_W, Sched<[WriteShuffleLd, ReadAfterLd]>; + XOP_4V, VEX_W, Sched<[WriteShuffleLd, ReadAfterLd, ReadAfterLd]>; def rmr : IXOPi8Reg opc, string OpcodeStr, SDNode OpNode, [(set VR128:$dst, (v16i8 (OpNode (vt128 VR128:$src1), (vt128 (bitconvert (loadv2i64 addr:$src2))), (vt128 VR128:$src3))))]>, - XOP_4V, Sched<[WriteShuffleLd, ReadAfterLd]>; + XOP_4V, Sched<[WriteShuffleLd, ReadAfterLd, + // 128mem:$src2 + ReadDefault, ReadDefault, ReadDefault, ReadDefault, + ReadDefault, + // VR128:$src3 + ReadAfterLd]>; // For disassembler let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in def rrr_REV : IXOPi8Reg opc, string OpcodeStr, RegisterClass RC, "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), [(set RC:$dst, (VT (or (and (load addr:$src3), RC:$src1), (X86andnp (load addr:$src3), RC:$src2))))]>, - XOP_4V, VEX_W, Sched<[WriteShuffleLd, ReadAfterLd]>; + XOP_4V, VEX_W, Sched<[WriteShuffleLd, ReadAfterLd, ReadAfterLd]>; def rmr : IXOPi8Reg, - XOP_4V, Sched<[WriteShuffleLd, ReadAfterLd]>; + XOP_4V, Sched<[WriteShuffleLd, ReadAfterLd, + // x86memop:$src2 + ReadDefault, ReadDefault, ReadDefault, ReadDefault, + ReadDefault, + // RC::$src3 + ReadAfterLd]>; // For disassembler let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in def rrr_REV : IXOPi8Reg Opc, string OpcodeStr, RegisterClass RC, (VT (X86vpermil2 RC:$src1, RC:$src2, (bitconvert (IntLdFrag addr:$src3)), (i8 imm:$src4))))]>, VEX_W, - Sched<[WriteFShuffleLd, ReadAfterLd]>; + Sched<[WriteFShuffleLd, ReadAfterLd, ReadAfterLd]>; def mr : IXOP5 Opc, string OpcodeStr, RegisterClass RC, [(set RC:$dst, (VT (X86vpermil2 RC:$src1, (FPLdFrag addr:$src2), RC:$src3, (i8 imm:$src4))))]>, - Sched<[WriteFShuffleLd, ReadAfterLd]>; + Sched<[WriteFShuffleLd, ReadAfterLd, + // fpmemop:$src2 + ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, + // RC:$src3 + ReadAfterLd]>; // For disassembler let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in def rr_REV : IXOP5