1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
//===- SeedCollection.cpp - Seed collection pass --------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/SandboxIR/Module.h"
#include "llvm/SandboxIR/Region.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
namespace llvm {
static cl::opt<unsigned>
OverrideVecRegBits("sbvec-vec-reg-bits", cl::init(0), cl::Hidden,
cl::desc("Override the vector register size in bits, "
"which is otherwise found by querying TTI."));
static cl::opt<bool>
AllowNonPow2("sbvec-allow-non-pow2", cl::init(false), cl::Hidden,
cl::desc("Allow non-power-of-2 vectorization."));
#define LoadSeedsDef "loads"
#define StoreSeedsDef "stores"
cl::opt<std::string> CollectSeeds(
"sbvec-collect-seeds", cl::init(StoreSeedsDef), cl::Hidden,
cl::desc("Collect these seeds. Use empty for none or a comma-separated "
"list of '" StoreSeedsDef "' and '" LoadSeedsDef "'."));
namespace sandboxir {
SeedCollection::SeedCollection(StringRef Pipeline)
: FunctionPass("seed-collection"),
RPM("rpm", Pipeline, SandboxVectorizerPassBuilder::createRegionPass) {}
bool SeedCollection::runOnFunction(Function &F, const Analyses &A) {
bool Change = false;
const auto &DL = F.getParent()->getDataLayout();
unsigned VecRegBits =
OverrideVecRegBits != 0
? OverrideVecRegBits
: A.getTTI()
.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
.getFixedValue();
bool CollectStores = CollectSeeds.find(StoreSeedsDef) != std::string::npos;
bool CollectLoads = CollectSeeds.find(LoadSeedsDef) != std::string::npos;
// TODO: Start from innermost BBs first
for (auto &BB : F) {
SeedCollector SC(&BB, A.getScalarEvolution(), CollectStores, CollectLoads);
for (SeedBundle &Seeds : SC.getStoreSeeds()) {
unsigned ElmBits =
Utils::getNumBits(VecUtils::getElementType(Utils::getExpectedType(
Seeds[Seeds.getFirstUnusedElementIdx()])),
DL);
auto DivideBy2 = [](unsigned Num) {
auto Floor = VecUtils::getFloorPowerOf2(Num);
if (Floor == Num)
return Floor / 2;
return Floor;
};
// Try to create the largest vector supported by the target. If it fails
// reduce the vector size by half.
for (unsigned SliceElms = std::min(VecRegBits / ElmBits,
Seeds.getNumUnusedBits() / ElmBits);
SliceElms >= 2u; SliceElms = DivideBy2(SliceElms)) {
if (Seeds.allUsed())
break;
// Keep trying offsets after FirstUnusedElementIdx, until we vectorize
// the slice. This could be quite expensive, so we enforce a limit.
for (unsigned Offset = Seeds.getFirstUnusedElementIdx(),
OE = Seeds.size();
Offset + 1 < OE; Offset += 1) {
// Seeds are getting used as we vectorize, so skip them.
if (Seeds.isUsed(Offset))
continue;
if (Seeds.allUsed())
break;
auto SeedSlice =
Seeds.getSlice(Offset, SliceElms * ElmBits, !AllowNonPow2);
if (SeedSlice.empty())
continue;
assert(SeedSlice.size() >= 2 && "Should have been rejected!");
// Create a region containing the seed slice.
auto &Ctx = F.getContext();
Region Rgn(Ctx, A.getTTI());
Rgn.setAux(SeedSlice);
// Run the region pass pipeline.
Change |= RPM.runOnRegion(Rgn, A);
Rgn.clearAux();
}
}
}
}
return Change;
}
} // namespace sandboxir
} // namespace llvm
|
