1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
//===--- NamedReturnValueOptimization.swift --------------------------------==//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
import SIL
/// Removes a `copy_addr` to an indirect out argument by replacing the source of the copy
/// (which must be an `alloc_stack`) with the out argument itself.
///
/// The following SIL pattern will be optimized:
///
/// sil @foo : $@convention(thin) <T> () -> @out T {
/// bb0(%0 : $*T):
/// %2 = alloc_stack $T
/// ...
/// copy_addr %some_value to [init] %2 // or any other writes to %2
/// ...
/// bbN:
/// copy_addr [take] %2 to [init] %0 : $*T // the only use of %0
/// ... // no writes
/// return
///
/// to:
///
/// sil @foo : $@convention(thin) <T> (@out T) -> () {
/// bb0(%0 : $*T):
/// %2 = alloc_stack $T // is dead now
/// ...
/// copy_addr %some_value to [init] %0
/// ...
/// bbN:
/// ...
/// return
///
/// This optimization can be done because we know that:
/// * The out argument dominates all uses of the copy_addr's source (because it's a function argument).
/// * It's not aliased (by definition). We can't allow aliases to be accessed between the initialization and the return.
///
/// This pass shouldn't run before serialization. It might prevent predictable memory optimizations
/// in a caller after inlining, because the memory location (the out argument = an alloc_stack in the caller)
/// might be written multiple times after this optimization.
///
let namedReturnValueOptimization = FunctionPass(name: "named-return-value-optimization") {
(function: Function, context: FunctionPassContext) in
for outArg in function.arguments[0..<function.numIndirectResultArguments] {
if let copyToArg = findCopyForNRVO(for: outArg) {
performNRVO(with: copyToArg, context)
}
}
}
/// Returns a copy_addr which copies from an alloc_stack to the `outArg` at the end of the function.
///
private func findCopyForNRVO(for outArg: FunctionArgument) -> CopyAddrInst? {
guard let singleArgUse = outArg.uses.ignoreDebugUses.singleUse,
let copyToArg = singleArgUse.instruction as? CopyAddrInst else {
return nil
}
assert(singleArgUse == copyToArg.destinationOperand,
"single use of out-argument cannot be the source of a copy")
// Don't perform NRVO unless the copy is a [take]. This is the easiest way
// to determine that the local variable has ownership of its value and ensures
// that removing a copy is a reference count neutral operation. For example,
// this copy can't be trivially eliminated without adding a retain.
// sil @f : $@convention(thin) (@guaranteed T) -> @out T
// bb0(%in : $*T, %out : $T):
// %local = alloc_stack $T
// store %in to %local : $*T
// copy_addr %local to [init] %out : $*T
if !copyToArg.isTakeOfSrc {
return nil
}
guard let sourceStackAlloc = copyToArg.source as? AllocStackInst else {
return nil
}
// NRVO for alloc_stack [dynamic_lifetime] will invalidate OSSA invariants.
if sourceStackAlloc.hasDynamicLifetime && copyToArg.parentFunction.hasOwnership {
return nil
}
if !(copyToArg.parentBlock.terminator is ReturnInst) {
return nil
}
// This check is overly conservative, because we only need to check if the source
// of the copy is not written to. But the copy to the out argument is usually the last
// instruction of the function, so it doesn't matter.
if isAnyInstructionWritingToMemory(after: copyToArg) {
return nil
}
return copyToArg
}
private func performNRVO(with copy: CopyAddrInst, _ context: FunctionPassContext) {
copy.source.replaceAllUsesExceptDealloc(with: copy.destination, context)
assert(copy.source == copy.destination)
context.erase(instruction: copy)
}
private func isAnyInstructionWritingToMemory(after: Instruction) -> Bool {
var followingInst = after.next
while let fi = followingInst {
if fi.mayWriteToMemory && !(fi is DeallocStackInst) {
return true
}
followingInst = fi.next
}
return false
}
private extension Value {
func replaceAllUsesExceptDealloc(with replacement: Value, _ context: some MutatingContext) {
uses.lazy.filter{!($0.instruction is Deallocation)}.replaceAll(with: replacement, context)
}
}
|
