File: multi_forward.go

package info (click to toggle)
fever 1.4.0-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 920 kB
  • sloc: sh: 41; makefile: 18
file content (262 lines) | stat: -rw-r--r-- 7,420 bytes parent folder | download | duplicates (4)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
package processing

// DCSO FEVER
// Copyright (c) 2021, DCSO GmbH

import (
	"net"
	"sync"
	"time"

	"github.com/DCSO/fever/types"
	"github.com/DCSO/fever/util"
	log "github.com/sirupsen/logrus"
)

// MultiForwardPerfStats contains performance stats written to InfluxDB
// for monitoring.
type MultiForwardPerfStats struct {
	Received     uint64 `influx:"output_received_per_sec"`
	Dropped      uint64 `influx:"output_dropped"`
	BufferLength uint64 `influx:"output_buffer_length"`
}

// MultiForwardOutput defines a single output target including socket path,
// whether to filter the output by event type and if so, what event types to let
// pass.
type MultiForwardOutput struct {
	Socket       string   `mapstructure:"socket"`
	All          bool     `mapstructure:"all"`
	BufferLength uint64   `mapstructure:"buffer-length"`
	Types        []string `mapstructure:"types"`
}

// MultiForwardConfiguration contains a setup for the multi-forwarder as read
// and parsed from the configuration file.
type MultiForwardConfiguration struct {
	Outputs      map[string]MultiForwardOutput `mapstructure:"multi-forward"`
	Shippers     []*MultiForwardShipper
	StatsEncoder *util.PerformanceStatsEncoder
}

// MultiForwardShipper is a concurrent, self-contained component that receives
// Entries from an input channel and writes the associated JSON to an output
// socket, filtering the output if desired. Also handles reconnection.
type MultiForwardShipper struct {
	OutputName          string
	Logger              *log.Entry
	ForwardInChan       chan types.Entry
	OutputSocket        string
	OutputConn          net.Conn
	Reconnecting        bool
	ReconnLock          sync.Mutex
	ReconnectNotifyChan chan bool
	StopReconnectChan   chan bool
	ReconnectTimes      int
	PerfStats           MultiForwardPerfStats
	StatsEncoder        *util.PerformanceStatsEncoder
	StopChan            chan bool
	StoppedChan         chan bool
	StopCounterChan     chan bool
	StoppedCounterChan  chan bool
	Running             bool
	Lock                sync.Mutex
}

func (mfs *MultiForwardShipper) reconnectForward() {
	for range mfs.ReconnectNotifyChan {
		var i int
		mfs.Logger.Infof("Reconnecting to forwarding socket (%s)...", mfs.OutputSocket)
		outputConn, myerror := net.Dial("unix", mfs.OutputSocket)
		mfs.ReconnLock.Lock()
		if !mfs.Reconnecting {
			mfs.Reconnecting = true
		} else {
			mfs.ReconnLock.Unlock()
			continue
		}
		mfs.ReconnLock.Unlock()

		for i = 0; (mfs.ReconnectTimes == 0 || i < mfs.ReconnectTimes) && myerror != nil; i++ {
			select {
			case <-mfs.StopReconnectChan:
				return
			default:
				mfs.Logger.WithFields(log.Fields{
					"retry":      i + 1,
					"maxretries": mfs.ReconnectTimes,
				}).Warnf("error connecting to output socket, retrying: %s", myerror)
				time.Sleep(10 * time.Second)
				outputConn, myerror = net.Dial("unix", mfs.OutputSocket)
			}
		}
		if myerror != nil {
			mfs.Logger.WithFields(log.Fields{
				"retries": i,
			}).Fatalf("permanent error connecting to output socket: %s", myerror)
			mfs.ReconnLock.Unlock()
		} else {
			if i > 0 {
				mfs.Logger.WithFields(log.Fields{
					"retry_attempts": i,
				}).Infof("connection to output socket successful")
			}
			mfs.Lock.Lock()
			mfs.OutputConn = outputConn
			mfs.Lock.Unlock()
			mfs.ReconnLock.Lock()
			mfs.Reconnecting = false
			mfs.ReconnLock.Unlock()
		}
	}
}

func (mfs *MultiForwardShipper) runForward() {
	var err error
	for {
		select {
		case <-mfs.StopChan:
			close(mfs.StoppedChan)
			return
		default:
			for item := range mfs.ForwardInChan {
				mfs.PerfStats.Received++
				select {
				case <-mfs.StopChan:
					close(mfs.StoppedChan)
					return
				default:
					mfs.ReconnLock.Lock()
					if mfs.Reconnecting {
						mfs.ReconnLock.Unlock()
						mfs.PerfStats.Dropped++
						continue
					}
					mfs.ReconnLock.Unlock()
					mfs.Lock.Lock()
					if mfs.OutputConn != nil {
						_, err = mfs.OutputConn.Write([]byte(item.JSONLine))
						if err != nil {
							mfs.OutputConn.Close()
							mfs.Lock.Unlock()
							log.Warn(err)
							mfs.ReconnectNotifyChan <- true
							continue
						}
						_, err = mfs.OutputConn.Write([]byte("\n"))
						if err != nil {
							mfs.OutputConn.Close()
							mfs.Lock.Unlock()
							mfs.Logger.Warn(err)
							continue
						}
					}
					mfs.Lock.Unlock()
				}
			}
		}
	}
}

func (mfs *MultiForwardShipper) runCounter() {
	sTime := time.Now()
	for {
		time.Sleep(500 * time.Millisecond)
		select {
		case <-mfs.StopCounterChan:
			close(mfs.StoppedCounterChan)
			return
		default:
			if mfs.StatsEncoder == nil || time.Since(sTime) < mfs.StatsEncoder.SubmitPeriod {
				continue
			}
			// Lock the current measurements for submission. Since this is a blocking
			// operation, we don't want this to depend on how long submitter.Submit()
			// takes but keep it independent of that. Hence we take the time to create
			// a local copy of the counter to be able to reset and release the live
			// one as quickly as possible.
			mfs.Lock.Lock()
			// Make our own copy of the current counter
			myStats := MultiForwardPerfStats{
				Dropped:      mfs.PerfStats.Dropped,
				Received:     mfs.PerfStats.Received / uint64(mfs.StatsEncoder.SubmitPeriod.Seconds()),
				BufferLength: uint64(len(mfs.ForwardInChan)),
			}
			// Reset live counter
			mfs.PerfStats.Received = 0
			// Release live counter to not block further events
			mfs.Lock.Unlock()

			mfs.StatsEncoder.SubmitWithTags(myStats, map[string]string{
				"output": mfs.OutputName,
			})
			sTime = time.Now()
		}
	}
}

// Run starts all concurrent aspects of the forwarder, reading from the input
// channel and distributing incoming events after setting up the shippers from
// the configuration.
func (m *MultiForwardConfiguration) Run(inChan <-chan types.Entry, reconnectTimes int) {
	outputMap := make(map[string][]*MultiForwardShipper)
	fwdAll := make([]*MultiForwardShipper, 0)
	for name, output := range m.Outputs {
		mfs := &MultiForwardShipper{
			OutputName:     name,
			OutputSocket:   output.Socket,
			ReconnectTimes: reconnectTimes,
			Logger: log.WithFields(log.Fields{
				"domain": "forward",
				"output": name,
			}),
			ReconnectNotifyChan: make(chan bool),
			StopReconnectChan:   make(chan bool),
			StatsEncoder:        m.StatsEncoder,
		}
		mfs.StopChan = make(chan bool)
		mfs.ForwardInChan = make(chan types.Entry, output.BufferLength)
		if output.All {
			fwdAll = append(fwdAll, mfs)
		} else {
			for _, outT := range output.Types {
				outputMap[outT] = append(outputMap[outT], mfs)
			}
		}
		mfs.StopCounterChan = make(chan bool)
		mfs.StoppedCounterChan = make(chan bool)
		go mfs.reconnectForward()
		mfs.ReconnectNotifyChan <- true
		go mfs.runForward()
		go mfs.runCounter()
	}
	go func() {
		for inEntry := range inChan {
			if len(fwdAll) > 0 {
				for _, shipper := range fwdAll {
					select {
					case shipper.ForwardInChan <- inEntry:
						//pass
					default:
						shipper.PerfStats.Dropped++
					}
				}
			}
			if shippers, ok := outputMap[inEntry.EventType]; ok {
				for _, shipper := range shippers {
					select {
					case shipper.ForwardInChan <- inEntry:
						//pass
					default:
						shipper.PerfStats.Dropped++
					}
				}
			}
		}
	}()
}

// SubmitStats registers a PerformanceStatsEncoder for runtime stats submission.
func (m *MultiForwardConfiguration) SubmitStats(sc *util.PerformanceStatsEncoder) {
	m.StatsEncoder = sc
}