1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// gccgo-specific support for GC.
package runtime
import (
"internal/goarch"
"unsafe"
)
// For gccgo, use go:linkname to export compiler-called functions.
//
//go:linkname gcWriteBarrier
// gcRoot is a single GC root: a variable plus a ptrmask.
//go:notinheap
type gcRoot struct {
decl unsafe.Pointer // Pointer to variable.
size uintptr // Size of variable.
ptrdata uintptr // Length of gcdata.
gcdata *uint8 // Pointer mask.
}
// gcRootList is the set of GC roots for a package.
// The next field is used to put this all into a linked list.
// count gives the real length of the array.
type gcRootList struct {
next *gcRootList
count int
roots [1 << 26]gcRoot
}
// roots is the list of GC roots for the program.
// The compiler keeps this variable itself off the list.
var gcRoots *gcRootList
// Slice containing pointers to all reachable gcRoot's sorted by
// starting address (generated at init time from 'gcRoots').
// The compiler also keeps this variable itself off the list.
// The storage backing this slice is allocated via persistentalloc(), the
// idea being that we don't want to treat the slice itself as a global
// variable, since it points to things that don't need to be scanned
// themselves.
var gcRootsIndex []*gcRoot
// rootradixsort performs an in-place radix sort of the 'arr' rootptr slice.
// Note: not a stable sort, however we expect it to be called only on slices
// with no duplicate entries, so this should not matter.
func rootradixsort(arr []*gcRoot, lo, hi int, bit uint) {
// Partition the array into two bins based on the values at the
// specified bit position: 0's bin (grown from the left) and and
// 1's bin (grown from the right). We keep two boundary markers,
// the 0's boundary "zbb" (which grows to the right) and the 1's
// boundary "obb" (which grows to the left). At each step we
// examine the bit for the right-of-ZBB element: if it is zero, we
// leave it in place and move the ZBB to the right. If the bit is
// not zero, then we swap the ZBB and OBB elements and move the
// OBB to the left. When this is done, the two partitions are then
// sorted using the next lower bit.
// 0's bin boundary, initially set to before the first element
zbb := lo - 1
// 1's bin boundary, set to just beyond the last element
obb := hi + 1
// mask to pick up bit of interest
bmask := uintptr(1) << bit
for obb-zbb > 1 {
zbbval := uintptr(arr[zbb+1].decl) & bmask
if zbbval == 0 {
// Move zbb one to the right
zbb++
} else {
// Move obb one to the left and swap
arr[obb-1], arr[zbb+1] = arr[zbb+1], arr[obb-1]
obb--
}
}
if bit != 0 {
// NB: in most cases there is just a single partition to visit
// so if we wanted to reduce stack space we could check for this
// and insert a goto back up to the top.
if zbb-lo > 0 {
rootradixsort(arr, lo, zbb, bit-1)
}
if hi-obb > 0 {
rootradixsort(arr, obb, hi, bit-1)
}
}
}
//go:nowritebarrier
func createGcRootsIndex() {
// Count roots
nroots := 0
gcr := gcRoots
for gcr != nil {
nroots += gcr.count
gcr = gcr.next
}
// Construct the gcRootsIndex slice. Use non-heap storage for the array
// backing the slice.
sp := (*notInHeapSlice)(unsafe.Pointer(&gcRootsIndex))
sp.array = (*notInHeap)(persistentalloc1(goarch.PtrSize*uintptr(nroots), goarch.PtrSize, &memstats.other_sys))
if sp.array == nil {
throw("runtime: cannot allocate memory")
}
sp.len = nroots
sp.cap = nroots
// Populate the roots index slice
gcr = gcRoots
k := 0
for gcr != nil {
for i := 0; i < gcr.count; i++ {
gcRootsIndex[k] = &gcr.roots[i]
k++
}
gcr = gcr.next
}
// Sort it by starting address.
rootradixsort(gcRootsIndex, 0, nroots-1, goarch.PtrSize*8-1)
}
// registerGCRoots is called by compiler-generated code.
//go:linkname registerGCRoots
// registerGCRoots is called by init functions to register the GC
// roots for a package. The init functions are run sequentially at
// the start of the program, so no locking is needed.
func registerGCRoots(r *gcRootList) {
r.next = gcRoots
gcRoots = r
}
// checkPreempt is called when the preempt field in the running G is true.
// It preempts the goroutine if it is safe to do so.
// If preemptscan is true, this scans the stack for the garbage collector
// and carries on.
func checkPreempt() {
gp := getg()
if !gp.preempt || gp != gp.m.curg || !canPreemptM(gp.m) {
return
}
if gp.preemptStop {
mcall(preemptPark)
}
// Act like goroutine called runtime.Gosched.
mcall(gopreempt_m)
}
// gcWriteBarrier implements a write barrier. This is implemented in
// assembly in the gc library, but there is no special advantage to
// doing so with gccgo.
//go:nosplit
//go:nowritebarrier
func gcWriteBarrier(dst *uintptr, src uintptr) {
buf := &getg().m.p.ptr().wbBuf
if !buf.putFast(src, *dst) {
wbBufFlush(dst, src)
}
*dst = src
}
|
