1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
module God
module Conditions
# Condition Symbol :flapping
# Type: Trigger
#
# Trigger when a Task transitions to or from a state or states a given number
# of times within a given period.
#
# Paramaters
# Required
# +times+ is the number of times that the Task must transition before
# triggering.
# +within+ is the number of seconds within which the Task must transition
# the specified number of times before triggering. You may use
# the sugar methods #seconds, #minutes, #hours, #days to clarify
# your code (see examples).
# --one or both of--
# +from_state+ is the state (as a Symbol) from which the transition must occur.
# +to_state is the state (as a Symbol) to which the transition must occur.
#
# Optional:
# +retry_in+ is the number of seconds after which to re-monitor the Task after
# it has been disabled by the condition.
# +retry_times+ is the number of times after which to permanently unmonitor
# the Task.
# +retry_within+ is the number of seconds within which
#
# Examples
#
# Trigger if
class Flapping < TriggerCondition
attr_accessor :times,
:within,
:from_state,
:to_state,
:retry_in,
:retry_times,
:retry_within
def initialize
self.info = "process is flapping"
end
def prepare
@timeline = Timeline.new(self.times)
@retry_timeline = Timeline.new(self.retry_times)
end
def valid?
valid = true
valid &= complain("Attribute 'times' must be specified", self) if self.times.nil?
valid &= complain("Attribute 'within' must be specified", self) if self.within.nil?
valid &= complain("Attributes 'from_state', 'to_state', or both must be specified", self) if self.from_state.nil? && self.to_state.nil?
valid
end
def process(event, payload)
begin
if event == :state_change
event_from_state, event_to_state = *payload
from_state_match = !self.from_state || self.from_state && Array(self.from_state).include?(event_from_state)
to_state_match = !self.to_state || self.to_state && Array(self.to_state).include?(event_to_state)
if from_state_match && to_state_match
@timeline << Time.now
concensus = (@timeline.size == self.times)
duration = (@timeline.last - @timeline.first) < self.within
if concensus && duration
@timeline.clear
trigger
retry_mechanism
end
end
end
rescue => e
puts e.message
puts e.backtrace.join("\n")
end
end
private
def retry_mechanism
if self.retry_in
@retry_timeline << Time.now
concensus = (@retry_timeline.size == self.retry_times)
duration = (@retry_timeline.last - @retry_timeline.first) < self.retry_within
if concensus && duration
# give up
Thread.new do
sleep 1
# log
msg = "#{self.watch.name} giving up"
applog(self.watch, :info, msg)
end
else
# try again later
Thread.new do
sleep 1
# log
msg = "#{self.watch.name} auto-reenable monitoring in #{self.retry_in} seconds"
applog(self.watch, :info, msg)
sleep self.retry_in
# log
msg = "#{self.watch.name} auto-reenabling monitoring"
applog(self.watch, :info, msg)
if self.watch.state == :unmonitored
self.watch.monitor
end
end
end
end
end
end
end
end
|
