1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
From 7c30e23f115ae285b497ef11af0153703111dff2 Mon Sep 17 00:00:00 2001
From: Valentin Churavy <v.churavy@gmail.com>
Date: Sun, 22 Dec 2019 14:25:50 -0500
Subject: [PATCH 1/2] [SelectionDAG] Copy FP flags when visiting a binary
instruction.
Summary:
We noticed in Julia that the sequence below no longer turned into
a sequence of FMA instructions in LLVM 7+, but it did in LLVM 6.
```
%29 = fmul contract <4 x double> %wide.load, %wide.load16
%30 = fmul contract <4 x double> %wide.load13, %wide.load17
%31 = fmul contract <4 x double> %wide.load14, %wide.load18
%32 = fmul contract <4 x double> %wide.load15, %wide.load19
%33 = fadd fast <4 x double> %vec.phi, %29
%34 = fadd fast <4 x double> %vec.phi10, %30
%35 = fadd fast <4 x double> %vec.phi11, %31
%36 = fadd fast <4 x double> %vec.phi12, %32
```
Unlike Clang, Julia doesn't set the `unsafe-fp-math=true` function
attribute, but rather emits more local instruction flags.
This partially undoes https://reviews.llvm.org/D46854 and if required I can try to minimize the test further.
Reviewers: spatel, mcberg2017
Reviewed By: spatel
Subscribers: chriselrod, merge_guards_bot, hiraditya, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D71495
---
.../SelectionDAG/SelectionDAGBuilder.cpp | 7 +++++
llvm/test/CodeGen/X86/fmf-reduction.ll | 26 +++++++++++++++++++
2 files changed, 33 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/fmf-reduction.ll
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index bfeb3d1bc2b..e6362c19691 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -2833,6 +2833,13 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
if (isVectorReductionOp(&I)) {
Flags.setVectorReduction(true);
LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
+
+ // If no flags are set we will propagate the incoming flags, if any flags
+ // are set, we will intersect them with the incoming flag and so we need to
+ // copy the FMF flags here.
+ if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) {
+ Flags.copyFMF(*FPOp);
+ }
}
SDValue Op1 = getValue(I.getOperand(0));
diff --git a/test/CodeGen/X86/fmf-reduction.ll b/test/CodeGen/X86/fmf-reduction.ll
new file mode 100644
index 00000000000..1d669d2a924
--- /dev/null
+++ b/test/CodeGen/X86/fmf-reduction.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=fma | FileCheck %s
+
+; Propagation of IR FMF should not drop flags when adding the DAG reduction flag.
+; This should include an FMA instruction, not separate FMUL/FADD.
+
+define double @julia_dotf(<4 x double> %x, <4 x double> %y, <4 x double> %z, i1 %t3) {
+; CHECK-LABEL: julia_dotf:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2
+; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm1
+; CHECK-NEXT: vaddpd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; CHECK-NEXT: vaddsd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %t1 = fmul contract <4 x double> %x, %y
+ %t2 = fadd fast <4 x double> %z, %t1
+ %rdx.shuf = shufflevector <4 x double> %t2, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+ %bin.rdx22 = fadd fast <4 x double> %t2, %rdx.shuf
+ %rdx.shuf23 = shufflevector <4 x double> %bin.rdx22, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+ %bin.rdx24 = fadd fast <4 x double> %bin.rdx22, %rdx.shuf23
+ %t4 = extractelement <4 x double> %bin.rdx24, i32 0
+ ret double %t4
+}
+
--
2.24.1
|
