File: rrd_utils.lua

package info (click to toggle)
ntopng 5.2.1%2Bdfsg1-2
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 121,832 kB
  • sloc: javascript: 143,431; cpp: 71,175; ansic: 11,108; sh: 4,687; makefile: 911; python: 587; sql: 512; pascal: 234; perl: 118; ruby: 52; exp: 4
file content (304 lines) | stat: -rw-r--r-- 9,351 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
local os_utils = require "os_utils"
require "lua_utils"
require "rrd_paths"

SECONDS_IN_A_HOUR = 3600
SECONDS_IN_A_DAY = SECONDS_IN_A_HOUR*24

local rrd_utils = {}

-- NOTE: DO NOT EXTEND this module. This is deprecated and will be removed.

--------------------------------------------------------------------------------

-- Note: these date functions are expensive, use with care!

function timestamp_to_date(ts)
  return os.date("*t", ts)
end

function date_to_timestamp(dt)
  return os.time(dt)
end

function date_month_tostring(m)
  return os.date("%B", date_to_timestamp(m))  
end

function date_day_tostring(d)
  local res = os.date("%d", date_to_timestamp(d))
  if d.day < 10 then
    res = string.sub(res, 2)
  end
  return res
end

function date_wday_tostring(d)
  return os.date("%A", date_to_timestamp(d))
end

-- returns the days in the given year
function date_get_year_days(y)
  return timestamp_to_date(os.time{year=y+1, day=1, month=1, hour=0} - 1).yday
end

function date_tostring(dt)
  return os.date("%x %X", dt)
end

--------------------------------------------------------------------------------

-- Considers NaN and negative values as 0
function rrd_get_positive_value(x)
  if(x ~= x) or (x <= 0) then
    return 0
  else
    return x
  end
end

--------------------------------------------------------------------------------

--
-- Integrates derivate RRD data to meet the specified resolution.
--
-- Parameters
--  epoch_start - the desired epoch start
--  epoch_end - the desired end epoch
--  resolution - the desired resolution
--  start - the raw start date
--  rawdata - the raw derived series data
--  npoints - number of points in each data serie
--  rawstep - rawdata internal step
--  extra - extra parameters, for additional functionalities
--      date_mode - if true, the resolution is interpreted as if you want dates to be
--              separated by this resolution step. This enables daylight checks to
--              to adjust actual time intervals, which are computational intensive because of
--              Lua dates functions usage.
--      with_activity - if true, a new column "activity" will be added to the output data, containing the
--              total activity time in seconds for the interval.
--
-- Returns
--   On success: a list of times,
--               the rawdata parameter will be modified in place
--   On error: a string containing some error message
--
function rrd_interval_integrate(epoch_start, epoch_end, resolution, start, rawdata, npoints, rawstep, extra)
  resolution = math.floor(resolution)
  extra = extra or {}
  local date_mode = extra.date_mode
  local with_activity = extra.with_activity
  local orig_resol = resolution

  -- check resolution consistency
  if rawstep > resolution then
    -- TODO i18n should be available
    --~ return {error=i18n('error_rrd_low_resolution', {prefs=ntop.getHttpPrefix().."/lua/admin/prefs.lua?tab=on_disk_rrds"})}
    return {error='error_rrd_low_resolution'}
  end

  if epoch_start < start  then
    -- TODO i18n
    return {error='error_rrd_starts_before'}
  end

  if((with_activity) and (rawdata.activity ~= nil)) then
    -- TODO i18n
    return {error='error_activity_column_exists'}
  end

  local function set_resolution(prevtime, newtime)
    if date_mode then
      -- handle daylight changes
      resolution = orig_resol
      local from_dst = timestamp_to_date(prevtime).isdst
      local to_dst = timestamp_to_date(newtime).isdst
      if from_dst and not to_dst then
        resolution = resolution + 3600
      elseif not from_dst and to_dst then
        -- 1 to avoid infinite loop
        resolution = math.max(resolution - 3600, 1)
      end
    end
  end

  -- functions to handle the n-dimensional counters
  local function create_counters()
    local counters = {}
    for sname,sdata in pairs(rawdata) do
      counters[sname] = 0
    end
    return counters
  end
  
  -- iterates the n-dimensional counters and calls the 1-dimension callback, passing current counter value and column
  -- callback should return the new counter value
  local function for_each_counter_do_update(counters, callback)
    for sname,svalue in pairs(rawdata) do
      counters[sname] = callback(counters[sname], sname)
    end
  end

  -- use same table to avoid allocation overhead; Please take care not to cross src_idx with dst_idx
  local src_idx = 1 -- this will be used to read the original data
  local dst_idx = 1 -- this will be used to write the integrated data, in order to avoid further allocations
  local integer_ctrs = create_counters()
  local time_sum = epoch_start

  -- Pre-declare callbacks to improve performance while looping
  local prefix_slice
  local traffic_in_step = 0

  local function c_dump_slice (value, col)
    -- Save the previous value to avoid overwriting it when (src_idx == dst_idx)
    local prev_val = rawdata[col][src_idx]
    -- Calculate the traffic belonging to previous step
    local prefix_traffic = prefix_slice * rawdata[col][src_idx]
    traffic_in_step = traffic_in_step + prefix_traffic

    -- Save into previous step
    rawdata[col][dst_idx] = value + prefix_traffic

    -- Update the counter with suffix traffic
    return prev_val - prefix_traffic
  end

  local function c_accumulate_partial (value, col)
    traffic_in_step = traffic_in_step + rawdata[col][src_idx]
    return value + rawdata[col][src_idx]
  end

  local function c_fill_remaining (value, col)
    rawdata[col][dst_idx] = value
    
    -- will zero integer_ctrs for next intervals
    return 0
  end
  
  local function c_fill_nil (value, col)
    rawdata[col][dst_idx] = nil
  end

  -- normalize derived data
  for sname,sdata in pairs(rawdata) do
    for t=1, #sdata do
      sdata[t] = rrd_get_positive_value(sdata[t]) * rawstep
    end
  end

  --~ io.write("\n\n")
  --~ io.write("\nsrc_idx="..src_idx.." dst_idx="..dst_idx.." fstep/res="..rawstep.."/"..resolution.." fstart/dstart="..start.."/"..epoch_start.."\n")

  -- when the was data starts before desired start
  if start < epoch_start then
    local toskip = math.min(math.ceil((epoch_start - start) / rawstep), npoints)
    local prevtime = start + rawstep * (toskip-1)
    local alignment = math.floor((1 - (epoch_start - prevtime) / rawstep))

    -- skip starting rows
    src_idx = toskip + 1
    for_each_counter_do_update(integer_ctrs, function (value, col)
      return value + rawdata[col][toskip] * alignment
    end)
  end

  local curtime = start + (src_idx-1) * rawstep   -- goes up with raw steps
  local times = {}
  local activity = {}
  local activity_secs = 0

  local function debug_me()
    io.write("\nsrc_idx="..src_idx.." dst_idx="..dst_idx.." fstep/res="..rawstep.."/"..resolution.." curtime/dstart="..curtime.."/"..epoch_start.."\n")
    tprint(rawdata)
  end

  --~ debug_me()
  --~ assert(curtime >= epoch_start)
  --~ assert(src_idx <= dst_idx)
  
  -- main integration
  while src_idx <= npoints and time_sum <= epoch_end do
    set_resolution(time_sum, curtime)
    traffic_in_step = 0
    
    if curtime + rawstep >= time_sum + resolution then
      local prefix_t = time_sum + resolution - curtime
      --~ tprint(prefix_t)

      -- Calculate the time fraction belonging to previous step
      prefix_slice = math.floor(prefix_t / rawstep)

      times[dst_idx] = time_sum
      for_each_counter_do_update(integer_ctrs, c_dump_slice)

      if with_activity then
        if traffic_in_step > 0 then
          activity_secs = activity_secs + prefix_t
        end

        activity[dst_idx] = activity_secs;
        activity_secs = 0
      end

      dst_idx = dst_idx + 1
      
      time_sum = time_sum + resolution
    else
      -- Accumulate partial slices of traffic
      for_each_counter_do_update(integer_ctrs, c_accumulate_partial)

      if(with_activity and (traffic_in_step > 0)) then
        activity_secs = activity_secs + rawstep
      end
    end

    curtime = curtime + rawstep
    src_idx = src_idx + 1
  end

  -- case RRD end is before epoch_end
  while time_sum <= epoch_end do
    -- Save integer_ctrs result
    set_resolution(curtime, time_sum)
    times[dst_idx] = time_sum
    if with_activity then activity[dst_idx] = activity_secs; activity_secs = 0 end
    --~ assert(dst_idx < src_idx)
    for_each_counter_do_update(integer_ctrs, c_fill_remaining)
    dst_idx = dst_idx + 1
    curtime = time_sum
    time_sum = time_sum + resolution
  end

  --~ debug_me()

  -- nullify remaining data to free table entries
  while dst_idx <= npoints do
    for_each_counter_do_update(integer_ctrs, c_fill_nil)
    dst_idx = dst_idx + 1
  end

  if with_activity then rawdata.activity = activity end
  return times
end

-- ########################################################

-- Read information about the disk space used by rrd (size is in bytes)
--! @param timeout the maxium time to compute the size
function rrd_utils.storageInfo(ifid, timeout)
  local dirs = ntop.getDirs()
  local paths = getRRDPaths()
  local info = { total = 0 }

  for _, path in pairs(paths) do
    local absolute_path = os_utils.fixPath(dirs.workingdir .. "/" .. ifid .. "/".. path .."/")
    info[path] = getFolderSize(absolute_path, timeout)
    info["total"] = info["total"] + info[path]
  end

  return info
end

-- ########################################################

return rrd_utils