1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691
|
#!/usr/bin/ruby
# SPDX-License-Identifier: GPL-2.0-only
#
# Observe irq and softirq in top fashion
# (c) 2014 <abc@telekom.ru>
# License: GPL-2.0-only.
require 'getoptlong'
require 'curses'
require 'stringio'
@imode = :both
@omode = :table
@color = true
@showrps = false
GetoptLong.new(
["--help", "-h", GetoptLong::NO_ARGUMENT],
["--batch", "-b", GetoptLong::NO_ARGUMENT],
["--delay", "-d", GetoptLong::REQUIRED_ARGUMENT],
["--top", "-t", GetoptLong::NO_ARGUMENT],
["--table", "-x", GetoptLong::NO_ARGUMENT],
["--soft", "-s", GetoptLong::NO_ARGUMENT],
["--softirq", GetoptLong::NO_ARGUMENT],
["--softirqs", GetoptLong::NO_ARGUMENT],
["--irq", "-i", GetoptLong::NO_ARGUMENT],
["--irqs", GetoptLong::NO_ARGUMENT],
["--reverse", "-r", GetoptLong::NO_ARGUMENT],
["--nocolor", "-C", GetoptLong::NO_ARGUMENT],
["--eth", "-e", "--pps", GetoptLong::NO_ARGUMENT],
["--rps", "-R", "--xps", GetoptLong::NO_ARGUMENT]
).each do |opt, arg|
case opt
when '--help'
puts " Shows interrupt rates (per second) per cpu."
puts " Also shows irq affinity ('.' for disabled cpus),"
puts " and rps/xps affinity ('+' rx, '-' tx, '*' tx/rx)."
puts " Can show packet rate per eth queue."
puts
puts " Usage: #{$0} [-h] [-d #{@delay}] [-b] [-t|-x] [-i|-s] [-r]"
puts " -d --delay=n refresh interval"
puts " -s --softirq select softirqs only"
puts " -i --irq select hardware irqs only"
puts " -e --eth show extra eth stats (from ethtool)"
puts " -R --rps enable display of rps/xps"
puts " -x --table output in table mode (default)"
puts " -t --top output in flat top mode"
puts " -b --batch output non-interactively"
puts " -r --reverse reverse sort order"
puts " -C --nocolor disable colors"
puts
puts " Rates marked as '.' is forbidden by smp_affinity mask."
exit 0
when '--reverse'
@reverse = !@reverse
when '--batch'
@batch = true
@reverse = !@reverse if @omode == :top
when '--delay'
@delay = arg.to_i
when '--top'
@omode = :top
when '--table'
@omode = :table
when /--irq/
@imode = :irq
when /--soft/
@imode = :soft
when /--pps/
@pps = true
when /--nocolor/
@color = false
when /--rps/
@showrps = !@showrps
end
end
if !@delay && ARGV[0].to_f > 0
@delay = ARGV.shift.to_f
else
@delay = 5
end
@count = ARGV.shift.to_f if ARGV[0].to_i > 0
def read_table(tag, file)
@cpus = []
lines = IO.readlines(file)
@cpus = lines[0].scan(/CPU\d+/)
@icpus = @cpus if tag == 'i'
lines[2..-1].each do |li|
irq, stat, desc = li.match(/^\s*(\S+):((?:\s+\d+)+)(.*)$/).captures
stat = stat.scan(/\d+/)
@irqs << [tag, irq, desc]
stat.each_with_index do |val, i|
# interruptsN, 's|i', irq'N', 'cpuX', 'descr...'
@stats << [val.to_i, tag, irq, @cpus[i], desc.strip]
end
end
end
def read_procstat
@cstat = {}
lines = IO.readlines("/proc/stat").grep(/^cpu\d+ /)
lines.each do |li|
c, *d = li.split(" ")
d = d.map {|e| e.to_i}
@cstat[c] = d
end
end
def read_affinity
@aff = {}
Dir.glob("/proc/irq/*/smp_affinity").each do |af|
irq = af[%r{\d+}].to_i
a = IO.read(af).strip.to_i(16)
@aff[irq] = a
end
end
# list ethernet devices
def net_devices_pci
Dir['/sys/class/net/*'].reject do |f|
f += "/device" unless File.symlink?(f)
if File.symlink?(f)
!(File.readlink(f) =~ %r{devices/pci})
else
false
end
end.map {|f| File.basename(f)}
end
@devlist = net_devices_pci
@devre = Regexp.union(@devlist)
def get_rps(desc)
@rps = @xps = 0
return unless @showrps
return if @devlist.empty?
dev = desc[/\b(#{@devre})\b/, 1]
return unless dev
return unless desc =~ /-(tx|rx)+-\d+/i
qnr = desc[/-(\d+)\s*$/, 1]
return unless qnr
begin
@rps = IO.read("/sys/class/net/#{dev}/queues/rx-#{qnr}/rps_cpus").hex if desc =~ /rx/i
@xps = IO.read("/sys/class/net/#{dev}/queues/tx-#{qnr}/xps_cpus").hex if desc =~ /tx/i
rescue
end
end
def calc_rps(cpu)
m = 0
m |= 1 if @rps & (1 << cpu) != 0
m |= 2 if @xps & (1 << cpu) != 0
" +-*".slice(m, 1)
end
# ethtool -S eth0
def ethtool_grab_stat(dev = nil)
unless dev
@esto = @est if @est
@est = Hash.new { |h,k| h[k] = Hash.new(&h.default_proc) }
@devlist = net_devices_pci
@devre = Regexp.union(@devlist)
# own time counter because this stat could be paused
@ehts = @ets if @ets
@ets = @ts
@edt = @ets - @ehts if @ehts
@devlist.each {|e| ethtool_grab_stat(e)}
return
end
h = Hash.new {|k,v| k[v] = Array.new}
t = `ethtool -S #{dev} 2>/dev/null`
return if t == ''
t.split("\n").map { |e|
e.split(':')
}.reject { |e|
!e[1]
}.each { |k,v|
k.strip!
v = v.strip.to_i
if k =~ /^.x_queue_(\d+)_/
t = k.split('_', 4)
qdir = t[0]
qnr = t[2]
qk = t[3]
@est[dev][qdir][qnr][qk] = v
else
@est[dev][k] = v
end
}
end
def e_queue_stat(dev, qdir, qnr, k)
n = @est[dev][qdir][qnr][k]
o = @esto[dev][qdir][qnr][k]
d = (n - o) / @edt
if d > 0
"%s:%d" % [qdir, d]
else
nil
end
end
def e_dev_stat(dev, k, ks)
n = @est[dev][k]
o = @esto[dev][k]
r = (n - o) / @edt
ks = k unless ks
"%s:%d" % [ks, r]
end
def e_queue_stat_err(dev, qdir, qnr)
r = []
ek = @est[dev][qdir][qnr].keys.reject{|e| e =~ /^(bytes|packets)$/}
ek.each do |k|
n = @est[dev][qdir][qnr][k]
o = @esto[dev][qdir][qnr][k]
d = n - o
r << "%s_%s:%d" % [qdir, k, d] if d.to_i > 0
end
r
end
# this is not rate
def e_dev_stat_sum(dev, rk, ks)
ek = @est[dev].keys.reject{|ek| !(ek =~ rk)}
n = ek.inject(0) {|sum,k| sum += @est[dev][k].to_i}
o = ek.inject(0) {|sum,k| sum += @esto[dev][k].to_i rescue 0}
r = (n - o)
if r > 0
"%s:%d" % [ks, r]
else
nil
end
end
def print_ethstat(desc)
return if @devlist.empty?
dev = desc[/\b(#{@devre})\b/, 1]
return unless dev
unless @esto && @est
print ' []'
return
end
t = []
if desc =~ /-(tx|rx)+-\d+/i
qnr = desc[/-(\d+)\s*$/, 1]
if qnr
if desc =~ /rx/i
t << e_queue_stat(dev, "rx", qnr, "packets")
t += e_queue_stat_err(dev, "rx", qnr)
end
if desc =~ /tx/i
t << e_queue_stat(dev, "tx", qnr, "packets")
t += e_queue_stat_err(dev, "tx", qnr)
end
end
else
t << e_dev_stat(dev, "rx_packets", 'rx')
t << e_dev_stat(dev, "tx_packets", 'tx')
t << e_dev_stat_sum(dev, /_err/, 'err')
t << e_dev_stat_sum(dev, /_drop/, 'drop')
end
t.delete(nil)
print ' [' + t.join(' ') + ']'
end
def grab_stat
# @h[istorical]
@hstats = @stats
@hcstat = @cstat
@hts = @ts
@stats = []
@irqs = []
@ts = Time.now
@dt = @ts - @hts if @hts
read_table 'i', "/proc/interrupts"
read_table 's', "/proc/softirqs"
read_affinity
read_procstat
ethtool_grab_stat if @pps
end
def calc_speed
s = []
# calc speed
h = Hash.new(0)
@hstats.each do |v, t, i, c, d|
h[[t, i, c]] = v
end
# output
@h = {}
@t = Hash.new(0) # rate per cpu
@w = Hash.new(0) # irqs per irqN
@s = @stats.map do |v, t, i, c, d|
rate = (v - h[[t, i, c]]) / @dt
@t[c] += rate if t == 'i'
@w[[t, i]] += (v - h[[t, i, c]])
@h[[t, i, c]] = rate
[rate, v, t, i, c, d]
end
end
def calc_cpu
@cBusy = Hash.new(0)
@cHIrq = Hash.new(0)
@cSIrq = Hash.new(0)
# user, nice, system, [3] idle, [4] iowait, irq, softirq, etc.
@cstat.each do |c, d|
d = d.zip(@hcstat[c]).map {|a, b| a - b}
c = c.upcase
sum = d.reduce(:+)
@cBusy[c] = 100 - (d[3] + d[4]).to_f / sum * 100
@cHIrq[c] = (d[5]).to_f / sum * 100
@cSIrq[c] = (d[6]).to_f / sum * 100
end
end
def show_top
@s.sort!.reverse!
@s.reverse! if @reverse
rej = nil
rej = 's' if @imode == :irq
rej = 'i' if @imode == :soft
@s.each do |s, v, t, i, c, d|
next if t == rej
if s > 0
print "%9.1f %s %s <%s> %s" % [s, c.downcase, t, i, d]
print_ethstat(d) if @pps
puts
end
end
end
@ifilter = {}
def show_interrupts
maxlen = 7
@irqs.reverse! if @reverse
print "%s %*s " % [" ", maxlen, " "]
@icpus.each { |c| print " %6s" % c }
puts
# load
print "%*s: " % [maxlen + 2, "cpuUtil"]
@icpus.each { |c| print " %6.1f" % @cBusy[c] }
puts " total CPU utilization %"
#
print "%*s: " % [maxlen + 2, "%irq"]
@icpus.each { |c| print " %6.1f" % @cHIrq[c] }
puts " hardware IRQ CPU util%"
print "%*s: " % [maxlen + 2, "%sirq"]
@icpus.each { |c| print " %6.1f" % @cSIrq[c] }
puts " software IRQ CPU util%"
# total
print "%*s: " % [maxlen + 2, "irqTotal"]
@icpus.each { |c| print " %6d" % @t[c] }
puts " total hardware IRQs"
rej = nil
rej = 's' if @imode == :irq
rej = 'i' if @imode == :soft
@irqs.each do |t, i, desc|
next if t == rej
# include incrementally and all eth
unless @ifilter[[t, i]] || @showall
next unless @w[[t, i]] > 0 || desc =~ /eth/
@ifilter[[t, i]] = true
end
print "%s %*s: " % [t.to_s, maxlen, i.slice(0, maxlen)]
rps = get_rps(desc)
@icpus.each do |c|
cpu = c[/\d+/].to_i
aff = @aff[i.to_i]
off = ((aff & 1 << cpu) ==0)? true : false if aff
fla = calc_rps(cpu)
begin
v = @h[[t, i, c]]
if v > 0 || !off
print "%6d%c" % [v, fla]
elsif aff
print "%6s%c" % [".", fla]
end
rescue
end
end
print desc
print_ethstat(desc) if @pps
puts
end
end
def select_output
if @omode == :top
show_top
else
show_interrupts
end
end
def curses_choplines(text)
cols = Curses.cols - 1
rows = Curses.lines - 2
lines = text.split("\n").map {|e| e.slice(0, cols)}.slice(0, rows)
text = lines.join("\n")
text << "\n" * (rows - lines.size) if lines.size < rows
text
end
def show_help
puts "irqtop help:"
puts
puts " In table view, cells marked with '.' mean this hw irq is"
puts " disabled via /proc/irq/<irq>/smp_affinity"
puts " Interactive keys:"
puts " i Toggle (hardware) irqs view"
puts " s Toggle software irqs (softirqs) view"
puts " e Show eth stat per queue"
puts " R Show rps/xps affinity"
puts " t Flat top display mode"
puts " x Table display mode"
puts " r Reverse rows order"
puts " c Toggle colors (for eth)"
puts " a Show lines with zero rate (all)"
puts " A Clear lines with zero rates"
puts " . Pause screen updating"
puts " h,? This help screen"
puts " q Quit."
puts " Any other key will update display."
puts
puts "Press any key to continue."
end
hostname = `hostname`.strip
#
grab_stat
sleep 0.5
COLOR_GREEN = "\033[0;32m"
COLOR_YELLOW = "\033[0;33m"
COLOR_CYAN = "\033[0;36m"
COLOR_RED = "\033[0;31m"
COLOR_OFF = "\033[m"
def tty_printline(t)
latr = nil # line color
if t =~ /-rx-/
latr = COLOR_GREEN
elsif t =~ /-tx-/
latr = COLOR_YELLOW
elsif t =~ /\beth/
latr = COLOR_CYAN
end
print latr if latr
if t =~ /cpuUtil:|irq:|sirq:/
# colorize percentage values
t.scan(/\s+\S+/) do |e|
eatr = nil
if e =~ /^\s*[\d.]+$/
if e.to_i >= 90
eatr = COLOR_RED
elsif e.to_i <= 10
eatr = COLOR_GREEN
else
eatr = COLOR_YELLOW
end
end
print eatr if eatr
print e
print (latr)? latr : COLOR_OFF if eatr
end
elsif latr && t =~ / \[[^\]]+\]$/
# colorize eth stats
print $`
print COLOR_OFF if latr
$&.scan(/(.*?)(\w+)(:)(\d+)/) do |e|
eatr = nil
case e[1]
when 'rx'
eatr = COLOR_GREEN
when 'tx'
eatr = COLOR_YELLOW
else
eatr = COLOR_RED
end
eatr = nil if e[3].to_i == 0
print e[0]
print eatr if eatr
print e[1..-1].join
print (latr)? latr : COLOR_OFF if eatr
end
print $'
else
print t
end
print COLOR_OFF if latr
puts
end
def tty_output
if @color
$stdout = StringIO.new
yield
$stdout.rewind
txt = $stdout.read
$stdout = STDOUT
txt.split("\n", -1).each do |li|
tty_printline(li)
end
else
yield
end
end
if @batch
@color = @color && $stdout.tty?
loop do
grab_stat
calc_speed
calc_cpu
puts "#{hostname} - irqtop - #{Time.now}"
tty_output {
select_output
}
$stdout.flush
break if @count && (@count -= 1) == 0
sleep @delay
end
exit 0
end
Curses.init_screen
Curses.start_color
Curses.cbreak
Curses.noecho
Curses.nonl
Curses.init_pair(1, Curses::COLOR_GREEN, Curses::COLOR_BLACK);
Curses.init_pair(2, Curses::COLOR_YELLOW, Curses::COLOR_BLACK);
Curses.init_pair(3, Curses::COLOR_CYAN, Curses::COLOR_BLACK);
Curses.init_pair(4, Curses::COLOR_RED, Curses::COLOR_BLACK);
$stdscr = Curses.stdscr
$stdscr.keypad(true)
def curses_printline(t)
latr = nil # line color
if t =~ /-rx-/
latr = Curses.color_pair(1)
elsif t =~ /-tx-/
latr = Curses.color_pair(2)
elsif t =~ /\beth/
latr = Curses.color_pair(3)
end
$stdscr.attron(latr) if latr
if t =~ /cpuUtil:|irq:|sirq:/
# colorize percentage values
t.scan(/\s+\S+/) do |e|
eatr = nil
if e =~ /^\s*[\d.]+$/
if e.to_i >= 90
eatr = Curses.color_pair(4)
elsif e.to_i <= 10
eatr = Curses.color_pair(1)
else
eatr = Curses.color_pair(2)
end
end
$stdscr.attron(eatr) if eatr
$stdscr.addstr("#{e}")
$stdscr.attroff(eatr) if eatr
end
elsif latr && t =~ / \[[^\]]+\]$/
# colorize eth stats
$stdscr.addstr($`)
$stdscr.attroff(latr) if latr
$&.scan(/(.*?)(\w+)(:)(\d+)/) do |e|
eatr = nil
case e[1]
when 'rx'
eatr = Curses.color_pair(1)
when 'tx'
eatr = Curses.color_pair(2)
else
eatr = Curses.color_pair(4)
end
eatr = nil if e[3].to_i == 0
$stdscr.addstr(e[0])
$stdscr.attron(eatr) if eatr
$stdscr.addstr(e[1..-1].join)
$stdscr.attroff(eatr) if eatr
end
$stdscr.addstr($' + "\n")
else
$stdscr.addstr("#{t}\n")
end
$stdscr.attroff(latr) if latr
end
def curses_output
$stdout = StringIO.new
yield
$stdout.rewind
text = $stdout.read
$stdout = STDOUT
txt = curses_choplines(text)
if @color
txt.split("\n", -1).each_with_index do |li, i|
$stdscr.setpos(i, 0)
curses_printline(li)
end
else
$stdscr.setpos(0, 0)
$stdscr.addstr(txt)
end
$stdscr.setpos(1, 0)
Curses.refresh
end
def curses_enter(text, echo = true)
$stdscr.setpos(1, 0)
$stdscr.addstr(text + "\n")
$stdscr.setpos(1, 0)
Curses.attron(Curses::A_BOLD)
$stdscr.addstr(text)
Curses.attroff(Curses::A_BOLD)
Curses.refresh
Curses.echo if echo
Curses.timeout = -1
line = Curses.getstr
Curses.noecho
line
end
loop do
grab_stat
calc_speed
calc_cpu
curses_output {
puts "#{hostname} - irqtop - #{Time.now}"
select_output
}
Curses.timeout = @delay * 1000
ch = Curses.getch.chr rescue nil
case ch
when "\f"
Curses.clear
when "q", "Z", "z"
break
when 'i'
@imode = (@imode == :both)? :soft : :both
when 's'
@imode = (@imode == :both)? :irq : :both
when 't'
@omode = (@omode == :top)? :table : :top
when 'x'
@omode = (@omode == :table)? :top : :table
when 'e', 'p'
@pps = !@pps
when 'r'
@reverse = !@reverse
when 'c'
@color = !@color
when 'A'
@ifilter = {}
when 'a'
@ifilter = {}
@showall = !@showall
when 'R'
@showrps = !@showrps
when '.'
curses_enter("Pause, press enter to to continue: ", false)
when 'd'
d = curses_enter("Enter display interval: ")
@delay = d.to_f if d.to_f > 0
when 'h', '?'
curses_output { show_help }
Curses.timeout = -1
ch = Curses.getch.chr rescue nil
break if ch == 'q'
end
end
|