File: alert_utils.lua

package info (click to toggle)
ntopng 2.4%2Bdfsg1-3
  • links: PTS, VCS
  • area: main
  • in suites: stretch
  • size: 15,888 kB
  • ctags: 8,091
  • sloc: cpp: 21,442; ansic: 10,999; sh: 1,627; makefile: 423; pascal: 312; ruby: 34; exp: 4
file content (511 lines) | stat: -rw-r--r-- 18,735 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
--
-- (C) 2014-15 - ntop.org
--

-- This file contains the description of all functions
-- used to trigger host alerts

local verbose = false

j = require("dkjson")
require "persistence"

function ndpival_bytes(json, protoname)
    key = "ndpiStats"

    -- Host
    if((json[key] == nil) or (json[key][protoname] == nil)) then
        if(verbose) then print("## ("..protoname..") Empty<br>\n") end
        return(0)
    else
        local v = json[key][protoname]["bytes"]["sent"]+json[key][protoname]["bytes"]["rcvd"]
        if(verbose) then print("##  ("..protoname..") "..v.."<br>\n") end
        return(v)
    end
end

function proto_bytes(old, new, protoname)
    return(ndpival_bytes(new, protoname)-ndpival_bytes(old, protoname))
end
-- =====================================================

function bytes(old, new)
    if(new["sent"] ~= nil) then
        -- Host
        return((new["sent"]["bytes"]+new["rcvd"]["bytes"])-(old["sent"]["bytes"]+old["rcvd"]["bytes"]))
    else
        -- Interface
        return(new["bytes"]-old["bytes"])
    end
end

function packets(old, new)
    if(new["sent"] ~= nil) then
        -- Host
        return((new["sent"]["packets"]+new["rcvd"]["packets"])-(old["sent"]["packets"]+old["rcvd"]["packets"]))
    else
        -- Interface
        return(new["packets"]-old["packets"])
    end
end

function idle(old, new)
      local diff = os.time()-new["seen.last"]
      return(diff)
end

function dns(old, new)   return(proto_bytes(old, new, "DNS")) end
function p2p(old, new)   return(proto_bytes(old, new, "eDonkey")+proto_bytes(old, new, "BitTorrent")+proto_bytes(old, new, "Skype")) end

function are_alerts_suppressed(observed)
    local suppressAlerts = ntop.getHashCache("ntopng.prefs.alerts", observed)
    if((suppressAlerts == "") or (suppressAlerts == nil) or (suppressAlerts == "true")) then
        return false  -- alerts are not suppressed
    else
        if(verbose) then print("Skipping alert check for("..address.."): disabled in preferences<br>\n") end
        return true -- alerts are suppressed
    end
end

alerts_granularity = {
    { "min", "Every Minute" },
    { "5mins", "Every 5 Minutes" },
    { "hour", "Hourly" },
    { "day", "Daily" }
}

alarmable_metrics = {'bytes', 'packets', 'dns', 'p2p', 'idle', 'ingress', 'egress', 'inner'}

default_re_arm_minutes = {
    ["min"]  = 1    ,
    ["5mins"]= 5    ,
    ["hour"] = 60   ,
    ["day"]  = 3600
}

alert_functions_description = {
    ["bytes"]   = "Bytes delta (sent + received)",
    ["packets"] = "Packets delta (sent + received)",
    ["dns"]     = "DNS traffic delta bytes (sent + received)",
    ["p2p"]     = "Peer-to-peer traffic delta bytes (sent + received)",
    ["idle"]    = "Idle time since last packet sent (seconds)",
}

network_alert_functions_description = {
    ["ingress"] = "Ingress Bytes delta",
    ["egress"]  = "Egress Bytes delta",
    ["inner"]   = "Inner Bytes delta",
}


function re_arm_alert(alarm_source, timespan, alarmed_metric)
    local alarm_string = alarm_source.."_"..timespan.."_"..alarmed_metric
    local re_arm_key = "alerts_re_arming_"..alarm_string
    local re_arm_minutes = ntop.getHashCache("ntopng.prefs.alerts_"..timespan.."_re_arm_minutes", alarm_source)
    if re_arm_minutes ~= "" then
        re_arm_minutes = tonumber(re_arm_minutes)
    else
        re_arm_minutes = default_re_arm_minutes[timespan]
    end
    if verbose then io.write('re_arm_minutes: '..re_arm_minutes..'\n') end
    -- we don't care about key contents, we just care about its exsistance
    ntop.setCache(re_arm_key, "dummy", re_arm_minutes * 60)
end

function is_alert_re_arming(alarm_source, timespan, alarmed_metric)
    local alarm_string = alarm_source.."_"..timespan.."_"..alarmed_metric
    local re_arm_key = "alerts_re_arming_"..alarm_string
    local is_rearming = ntop.getCache(re_arm_key)
    if is_rearming ~= "" then
        if verbose then io.write('re_arm_key: '..re_arm_key..' -> ' ..is_rearming..'-- \n') end
        return true
    end
    return false
end

-- #################################################################
function delete_re_arming_alerts(alert_source)
    for k1, timespan in pairs(alerts_granularity) do
        timespan = timespan[1]
        local alarm_string = alert_source.."_"..timespan
        for k2, alarmed_metric in pairs(alarmable_metrics) do
            alarm_string = alarm_string.."_"..alarmed_metric
            local re_arm_key = "alerts_re_arming_"..alarm_string
            ntop.delCache(re_arm_key)
        end
    end
end


function delete_alert_configuration(alert_source)
    delete_re_arming_alerts(alert_source)
    for k1,timespan in pairs(alerts_granularity) do
        timespan = timespan[1]
        local key = "ntopng.prefs.alerts_"..timespan
        local alarms = ntop.getHashCache(key, alert_source)
        if alarms ~= "" then
            for k1, alarmed_metric in pairs(alarmable_metrics) do
                 if ntop.isPro() then
                     ntop.withdrawNagiosAlert(alert_source, timespan, alarmed_metric, "OK, alarm deactivated")
                 end
            end
        ntop.delHashCache(key, alert_source)
        key = "ntopng.prefs.alerts_"..timespan.."_re_arm_minutes"
        ntop.delHashCache(key, alert_source)
        end
    end
end


function check_host_alert(ifname, hostname, mode, key, old_json, new_json)
    if(verbose) then
        print("check_host_alert("..ifname..", "..hostname..", "..mode..", "..key..")<br>\n")

        print("<p>--------------------------------------------<p>\n")
        print("NEW<br>"..new_json.."<br>\n")
        print("<p>--------------------------------------------<p>\n")
        print("OLD<br>"..old_json.."<br>\n")
        print("<p>--------------------------------------------<p>\n")
    end

    old = j.decode(old_json, 1, nil)
    new = j.decode(new_json, 1, nil)

    -- str = "bytes;>;123,packets;>;12"
    hkey = "ntopng.prefs.alerts_"..mode

    str = ntop.getHashCache(hkey, hostname)

    -- if(verbose) then ("--"..hkey.."="..str.."--<br>") end
    if((str ~= nil) and (str ~= "")) then
        tokens = split(str, ",")

        for _,s in pairs(tokens) do
            -- if(verbose) then ("<b>"..s.."</b><br>\n") end
            t = string.split(s, ";")

            if(t[2] == "gt") then
                op = ">"
            else
                if(t[2] == "lt") then
                    op = "<"
                else
                    op = "=="
                end
            end

            local what = "val = "..t[1].."(old, new); if(val ".. op .. " " .. t[3] .. ") then return(true) else return(false) end"
            local f = loadstring(what)
            local rc = f()


            if(rc) then
                local alert_msg = "Threshold <b>"..t[1].."</b> crossed by host <A HREF="..ntop.getHttpPrefix().."/lua/host_details.lua?host="..key..">"..key.."</A> [".. val .." ".. op .. " " .. t[3].."]"
                local alert_level  = 1 -- alert_level_warning
                local alert_status = 1 -- alert_on
                local alert_type   = 2 -- alert_threshold_exceeded

                -- only if the alert is not in its re-arming period...
                if not is_alert_re_arming(key, mode, t[1]) then
                    if verbose then io.write("queuing alert\n") end
                    -- re-arm the alert
                    re_arm_alert(key, mode, t[1])
                    -- and send it to ntopng
                    ntop.queueAlert(alert_level, alert_status, alert_type, alert_msg)
                    if ntop.isPro() then
                        -- possibly send the alert to nagios as well
                        ntop.sendNagiosAlert(key, mode, t[1], alert_msg)
                    end
                else
                    if verbose then io.write("alarm silenced, re-arm in progress\n") end
                end

                if(verbose) then print("<font color=red>".. alert_msg .."</font><br>\n") end
            else  -- alert has not been triggered
                if(verbose) then print("<p><font color=green><b>Threshold "..t[1].."@"..key.." not crossed</b> [value="..val.."]["..op.." "..t[3].."]</font><p>\n") end
                if ntop.isPro() and not is_alert_re_arming(key, mode, t[1]) then
                    ntop.withdrawNagiosAlert(key, mode, t[1], "service OK")
                end
            end
        end
    end
end


function check_network_alert(ifname, network_name, mode, key, old_table, new_table)
    if(verbose) then
        io.write("check_newtowrk_alert("..ifname..", "..network_name..", "..mode..", "..key..")\n")
        io.write("new:\n")
        tprint(new_table)
        io.write("old:\n")
        tprint(old_table)
    end

    deltas = {}
    local delta_names = {'ingress', 'egress', 'inner'}
    for i = 1, 3 do
        local delta_name = delta_names[i]
        deltas[delta_name] = 0
        if old_table[delta_name] and new_table[delta_name] then
            deltas[delta_name] = new_table[delta_name] - old_table[delta_name]
        end
    end
    -- str = "bytes;>;123,packets;>;12"
    hkey = "ntopng.prefs.alerts_"..mode

    local str = ntop.getHashCache(hkey, network_name)

    -- if(verbose) then ("--"..hkey.."="..str.."--<br>") end
    if((str ~= nil) and (str ~= "")) then
        local tokens = split(str, ",")

        for _,s in pairs(tokens) do
            -- if(verbose) then ("<b>"..s.."</b><br>\n") end
            local t = string.split(s, ";")

            if(t[2] == "gt") then
                op = ">"
            else
                if(t[2] == "lt") then
                    op = "<"
                else
                    op = "=="
                end
            end

            local what = "val = deltas['"..t[1].."']; if(val ".. op .. " " .. t[3] .. ") then return(true) else return(false) end"
            local f = loadstring(what)
            local rc = f()


            if(rc) then
                local alert_msg = "Threshold <b>"..t[1].."</b> crossed by network <A HREF="..ntop.getHttpPrefix().."/lua/network_details.lua?network="..key.."&page=historical>"..network_name.."</A> [".. val .." ".. op .. " " .. t[3].."]"
                local alert_level = 1 -- alert_level_warning
                local alert_status = 1 -- alert_on
                local alert_type = 2 -- alert_threshold_exceeded

                if not is_alert_re_arming(network_name, mode, t[1]) then
                    if verbose then io.write("queuing alert\n") end
                    re_arm_alert(network_name, mode, t[1])
                    ntop.queueAlert(alert_level, alert_status, alert_type, alert_msg)
                    if ntop.isPro() then
                        -- possibly send the alert to nagios as well
                        ntop.sendNagiosAlert(network_name, mode, t[1], alert_msg)
                    end
                else
                    if verbose then io.write("alarm silenced, re-arm in progress\n") end
                end
                if(verbose) then print("<font color=red>".. alert_msg .."</font><br>\n") end
            else
                if(verbose) then print("<p><font color=green><b>Network threshold "..t[1].."@"..network_name.." not crossed</b> [value="..val.."]["..op.." "..t[3].."]</font><p>\n") end
                if ntop.isPro() and not is_alert_re_arming(network_name, mode, t[1]) then
                    ntop.withdrawNagiosAlert(network_name, mode, t[1], "service OK")
                end
            end
        end
    end
end

-- #################################

function check_interface_alert(ifname, mode, old_table, new_table)
   local ifname_clean = "iface_"..tostring(getInterfaceId(ifname))
    if(verbose) then
        print("check_interface_alert("..ifname..", "..mode..")<br>\n")
    end

    -- Needed because Lua. loadstring() won't work otherwise.
    old = old_table
    new = new_table

    -- str = "bytes;>;123,packets;>;12"
    hkey = "ntopng.prefs.alerts_"..mode

    str = ntop.getHashCache(hkey, ifname_clean)

    -- if(verbose) then ("--"..hkey.."="..str.."--<br>") end
    if((str ~= nil) and (str ~= "")) then
        tokens = split(str, ",")

        for _,s in pairs(tokens) do
            -- if(verbose) then ("<b>"..s.."</b><br>\n") end
            t = string.split(s, ";")

            if(t[2] == "gt") then
                op = ">"
            else
                if(t[2] == "lt") then
                    op = "<"
                else
                    op = "=="
                end
            end

            local what = "val = "..t[1].."(old, new); if(val ".. op .. " " .. t[3] .. ") then return(true) else return(false) end"
            local f = loadstring(what)
            local rc = f()

            if(rc) then
	       local alert_msg = "Threshold <b>"..t[1].."</b> crossed by interface <A HREF="..ntop.getHttpPrefix().."/lua/if_stats.lua?ifId="..tostring(getInterfaceId(ifname))..
                ">"..ifname.."</A> [".. val .." ".. op .. " " .. t[3].."]"
                local alert_level = 1 -- alert_level_warning
                local alert_status = 1 -- alert_on
                local alert_type = 2 -- alert_threshold_exceeded

                if not is_alert_re_arming(ifname_clean, mode, t[1]) then
                    if verbose then io.write("queuing alert\n") end
                    re_arm_alert(ifname_clean, mode, t[1])
                    ntop.queueAlert(alert_level, alert_status, alert_type, alert_msg)
                    if ntop.isPro() then
                        -- possibly send the alert to nagios as well
                        ntop.sendNagiosAlert(ifname_clean, mode, t[1], alert_msg)
                    end
                else
                    if verbose then io.write("alarm silenced, re-arm in progress\n") end
                end

                if(verbose) then print("<font color=red>".. alert_msg .."</font><br>\n") end
            else
                if(verbose) then print("<p><font color=green><b>Threshold "..t[1].."@"..ifname.." not crossed</b> [value="..val.."]["..op.." "..t[3].."]</font><p>\n") end
                if ntop.isPro() and not is_alert_re_arming(ifname_clean, mode, t[1]) then
                    ntop.withdrawNagiosAlert(ifname_clean, mode, t[1], "service OK")
                end
            end
        end
    end
end


-- #################################

function check_interface_threshold(ifname, mode)
    interface.select(ifname)
    local ifstats = aggregateInterfaceStats(interface.getStats())
    ifname_id = ifstats.id

    if are_alerts_suppressed("iface_"..ifname_id) then return end

    if(verbose) then print("check_interface_threshold("..ifname_id..", "..mode..")<br>\n") end
    basedir = fixPath(dirs.workingdir .. "/" .. ifname_id .. "/json/" .. mode)
    if(not(ntop.exists(basedir))) then
        ntop.mkdir(basedir)
    end

    --if(verbose) then print(basedir.."<br>\n") end
    interface.select(ifname)
    ifstats = aggregateInterfaceStats(interface.getStats())

    if (ifstats ~= nil) then
        fname = fixPath(basedir.."/iface_"..ifname_id.."_lastdump")

        if(verbose) then print(fname.."<p>\n") end
        if (ntop.exists(fname)) then
            -- Read old version
            old_dump = persistence.load(fname)
            if (old_dump ~= nil) then
                check_interface_alert(ifname, mode, old_dump, ifstats)
            end
        end

        -- Write new version
        persistence.store(fname, ifstats)
    end
end


function check_networks_threshold(ifname, mode)
    interface.select(ifname)
    local subnet_stats = interface.getNetworksStats()
    alarmed_subnets = ntop.getHashKeysCache("ntopng.prefs.alerts_"..mode)
    local ifname_id = interface.getStats().id

    local basedir = fixPath(dirs.workingdir .. "/" .. ifname_id .. "/json/" .. mode)
    if not ntop.exists(basedir) then
        ntop.mkdir(basedir)
    end

    for subnet,sstats in pairs(subnet_stats) do
        if sstats == nil or (alarmed_subnets and alarmed_subnets[subnet] == nil) or are_alerts_suppressed(subnet) then goto continue end
        local statspath = getPathFromKey(subnet)
        statspath = fixPath(basedir.. "/" .. statspath)
        if not ntop.exists(statspath) then
            ntop.mkdir(statspath)
        end
        statspath = fixPath(statspath .. "/alarmed_subnet_stats_lastdump")

        if ntop.exists(statspath) then
            -- Read old version
            old_dump = persistence.load(statspath)
            if (old_dump ~= nil) then
                -- (ifname, network_name, mode, key, old_table, new_table)
                check_network_alert(ifname, subnet, mode, sstats['network_id'], old_dump, subnet_stats[subnet])
            end
        end
        persistence.store(statspath, subnet_stats[subnet])
        ::continue::
    end
end

-- #################################

function check_host_threshold(ifname, host_ip, mode)
    interface.select(ifname)
    local ifstats = aggregateInterfaceStats(interface.getStats())
    ifname_id = ifstats.id

    if are_alerts_suppressed(host_ip) then return end

    if(verbose) then print("check_host_threshold("..ifname_id..", "..host_ip..", "..mode..")<br>\n") end
    basedir = fixPath(dirs.workingdir .. "/" .. ifname_id .. "/json/" .. mode)
    if(not(ntop.exists(basedir))) then
        ntop.mkdir(basedir)
    end

    json = interface.getHostInfo(host_ip)

    if(json ~= nil) then
        fname = fixPath(basedir.."/".. host_ip ..".json")

        if(verbose) then print(fname.."<p>\n") end
        -- Read old version
        f = io.open(fname, "r")
        if(f ~= nil) then
            old_json = f:read("*all")
            f:close()
            check_host_alert(ifname, host_ip, mode, host_ip, old_json, json["json"])
        end

        -- Write new version
        f = io.open(fname, "w")

        if(f ~= nil) then
            f:write(json["json"])
            f:close()
        end
    end
end

-- #################################

function scanAlerts(granularity, ifname)
   if(verbose) then print("[minute.lua] Scanning ".. granularity .." alerts for interface " .. ifname.."<p>\n") end

   check_interface_threshold(ifname, granularity)
   check_networks_threshold(ifname, granularity)
   -- host alerts checks
   local hash_key = "ntopng.prefs.alerts_"..granularity
   local hosts = ntop.getHashKeysCache(hash_key)
   if(hosts ~= nil) then
      for h in pairs(hosts) do
	 if(verbose) then print("[minute.lua] Checking host " .. h.." alerts<p>\n") end
	 check_host_threshold(ifname, h, granularity)
      end
   end
   -- network alerts checks
   if(networks ~= nil) then
      for n in pairs(networks) do
	 if(verbose) then print("[minute.lua] Checking network " .. h.." alerts<p>\n") end
      end
   end
end