You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
772 lines
19 KiB
772 lines
19 KiB
Demonstrations of offcputime, the Linux eBPF/bcc version.
|
|
|
|
|
|
This program shows stack traces that were blocked, and the total duration they
|
|
were blocked. It works by tracing when threads block and when they return to
|
|
CPU, measuring both the time they were blocked (aka the "off-CPU time") and the
|
|
blocked stack trace and the task name. This data is summarized in kernel by
|
|
summing the blocked time by unique stack trace and task name.
|
|
|
|
Here is some example output. The -K option was used to only match kernel stacks.
|
|
To explain what we are seeing: the very first stack trace looks like a page
|
|
fault (do_page_fault() etc) from the "chmod" command, and in total was off-CPU
|
|
for 13 microseconds.
|
|
|
|
# ./offcputime -K
|
|
Tracing off-CPU time (us) of all threads by kernel stack... Hit Ctrl-C to end.
|
|
^C
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
bit_wait_io
|
|
__wait_on_bit
|
|
wait_on_page_bit_killable
|
|
__lock_page_or_retry
|
|
filemap_fault
|
|
__do_fault
|
|
handle_mm_fault
|
|
__do_page_fault
|
|
do_page_fault
|
|
page_fault
|
|
chmod
|
|
13
|
|
|
|
schedule
|
|
rcu_nocb_kthread
|
|
kthread
|
|
ret_from_fork
|
|
ddebug_tables
|
|
rcuos/0
|
|
22
|
|
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
bit_wait_io
|
|
__wait_on_bit_lock
|
|
__lock_page
|
|
lock_page
|
|
__do_fault
|
|
handle_mm_fault
|
|
__do_page_fault
|
|
do_page_fault
|
|
page_fault
|
|
run
|
|
27
|
|
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
bit_wait_io
|
|
__wait_on_bit
|
|
wait_on_page_bit_killable
|
|
__lock_page_or_retry
|
|
filemap_fault
|
|
__do_fault
|
|
handle_mm_fault
|
|
__do_page_fault
|
|
do_page_fault
|
|
page_fault
|
|
clear_user
|
|
padzero
|
|
load_elf_binary
|
|
search_binary_handler
|
|
load_script
|
|
search_binary_handler
|
|
do_execveat_common.isra.27
|
|
run
|
|
28
|
|
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
bit_wait_io
|
|
__wait_on_bit
|
|
wait_on_page_bit_killable
|
|
__lock_page_or_retry
|
|
filemap_fault
|
|
__do_fault
|
|
handle_mm_fault
|
|
__do_page_fault
|
|
do_page_fault
|
|
page_fault
|
|
run
|
|
82
|
|
|
|
schedule
|
|
pipe_wait
|
|
pipe_read
|
|
__vfs_read
|
|
vfs_read
|
|
sys_read
|
|
entry_SYSCALL_64_fastpath
|
|
bash
|
|
94
|
|
|
|
schedule
|
|
rcu_gp_kthread
|
|
kthread
|
|
ret_from_fork
|
|
ddebug_tables
|
|
rcu_sched
|
|
104
|
|
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
bit_wait_io
|
|
__wait_on_bit
|
|
out_of_line_wait_on_bit
|
|
__wait_on_buffer
|
|
jbd2_journal_commit_transaction
|
|
kjournald2
|
|
kthread
|
|
ret_from_fork
|
|
mb_cache_list
|
|
jbd2/xvda1-8
|
|
986
|
|
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
bit_wait_io
|
|
__wait_on_bit
|
|
out_of_line_wait_on_bit
|
|
__wait_on_buffer
|
|
jbd2_journal_commit_transaction
|
|
kjournald2
|
|
kthread
|
|
ret_from_fork
|
|
mb_cache_list
|
|
jbd2/xvda1-8
|
|
6630
|
|
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
bit_wait_io
|
|
__wait_on_bit
|
|
out_of_line_wait_on_bit
|
|
do_get_write_access
|
|
jbd2_journal_get_write_access
|
|
__ext4_journal_get_write_access
|
|
ext4_mb_mark_diskspace_used
|
|
ext4_mb_new_blocks
|
|
ext4_ext_map_blocks
|
|
ext4_map_blocks
|
|
ext4_writepages
|
|
do_writepages
|
|
__filemap_fdatawrite_range
|
|
filemap_flush
|
|
ext4_alloc_da_blocks
|
|
ext4_rename
|
|
ext4_rename2
|
|
supervise
|
|
6645
|
|
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
bit_wait_io
|
|
__wait_on_bit
|
|
out_of_line_wait_on_bit
|
|
do_get_write_access
|
|
jbd2_journal_get_write_access
|
|
__ext4_journal_get_write_access
|
|
__ext4_new_inode
|
|
ext4_create
|
|
vfs_create
|
|
path_openat
|
|
do_filp_open
|
|
do_sys_open
|
|
sys_open
|
|
entry_SYSCALL_64_fastpath
|
|
supervise
|
|
12702
|
|
|
|
schedule
|
|
rcu_nocb_kthread
|
|
kthread
|
|
ret_from_fork
|
|
rcuos/2
|
|
16036
|
|
|
|
schedule
|
|
rcu_nocb_kthread
|
|
kthread
|
|
ret_from_fork
|
|
rcuos/4
|
|
24085
|
|
|
|
schedule
|
|
do_wait
|
|
sys_wait4
|
|
entry_SYSCALL_64_fastpath
|
|
run
|
|
233055
|
|
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
bit_wait_io
|
|
__wait_on_bit
|
|
wait_on_page_bit
|
|
truncate_inode_pages_range
|
|
truncate_inode_pages_final
|
|
ext4_evict_inode
|
|
evict
|
|
iput
|
|
__dentry_kill
|
|
dput
|
|
sys_rename
|
|
entry_SYSCALL_64_fastpath
|
|
supervise
|
|
297113
|
|
|
|
schedule
|
|
schedule_timeout
|
|
wait_woken
|
|
n_tty_read
|
|
tty_read
|
|
__vfs_read
|
|
vfs_read
|
|
sys_read
|
|
entry_SYSCALL_64_fastpath
|
|
bash
|
|
1789866
|
|
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
do_blockdev_direct_IO
|
|
__blockdev_direct_IO
|
|
blkdev_direct_IO
|
|
generic_file_read_iter
|
|
blkdev_read_iter
|
|
__vfs_read
|
|
vfs_read
|
|
sys_read
|
|
entry_SYSCALL_64_fastpath
|
|
dd
|
|
3310763
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
watchdog/1
|
|
3999989
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
watchdog/5
|
|
3999995
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
watchdog/4
|
|
3999996
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
watchdog/0
|
|
3999996
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
watchdog/3
|
|
3999998
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
watchdog/7
|
|
3999999
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
watchdog/2
|
|
4000001
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
watchdog/6
|
|
4000001
|
|
|
|
schedule
|
|
do_wait
|
|
sys_wait4
|
|
entry_SYSCALL_64_fastpath
|
|
bash
|
|
4039675
|
|
|
|
schedule
|
|
do_nanosleep
|
|
hrtimer_nanosleep
|
|
sys_nanosleep
|
|
entry_SYSCALL_64_fastpath
|
|
svscan
|
|
5000112
|
|
|
|
schedule
|
|
schedule_hrtimeout_range_clock
|
|
schedule_hrtimeout_range
|
|
poll_schedule_timeout
|
|
do_select
|
|
core_sys_select
|
|
sys_select
|
|
entry_SYSCALL_64_fastpath
|
|
snmpd
|
|
5998761
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
migration/3
|
|
6149779
|
|
|
|
schedule
|
|
schedule_hrtimeout_range_clock
|
|
schedule_hrtimeout_range
|
|
poll_schedule_timeout
|
|
do_select
|
|
core_sys_select
|
|
sys_select
|
|
entry_SYSCALL_64_fastpath
|
|
ntpd
|
|
6999832
|
|
|
|
schedule
|
|
worker_thread
|
|
kthread
|
|
ret_from_fork
|
|
kworker/u16:2
|
|
7131941
|
|
|
|
schedule
|
|
worker_thread
|
|
kthread
|
|
ret_from_fork
|
|
kworker/3:0
|
|
7999844
|
|
|
|
schedule
|
|
worker_thread
|
|
kthread
|
|
ret_from_fork
|
|
kworker/1:1
|
|
7999872
|
|
|
|
schedule
|
|
worker_thread
|
|
kthread
|
|
ret_from_fork
|
|
kworker/2:1
|
|
7999889
|
|
|
|
schedule
|
|
worker_thread
|
|
kthread
|
|
ret_from_fork
|
|
kworker/5:1
|
|
7999936
|
|
|
|
schedule
|
|
worker_thread
|
|
kthread
|
|
ret_from_fork
|
|
kworker/7:1
|
|
7999938
|
|
|
|
schedule
|
|
worker_thread
|
|
kthread
|
|
ret_from_fork
|
|
kworker/6:1
|
|
7999940
|
|
|
|
schedule
|
|
do_nanosleep
|
|
hrtimer_nanosleep
|
|
sys_nanosleep
|
|
entry_SYSCALL_64_fastpath
|
|
tail
|
|
8000905
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
migration/7
|
|
8197046
|
|
|
|
schedule
|
|
pipe_wait
|
|
pipe_read
|
|
__vfs_read
|
|
vfs_read
|
|
sys_read
|
|
entry_SYSCALL_64_fastpath
|
|
readproctitle
|
|
8197835
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
migration/4
|
|
8201851
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
migration/2
|
|
8203375
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
migration/6
|
|
8208664
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
migration/5
|
|
8209819
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
ddebug_tables
|
|
migration/0
|
|
8211292
|
|
|
|
schedule
|
|
smpboot_thread_fn
|
|
kthread
|
|
ret_from_fork
|
|
migration/1
|
|
8212100
|
|
|
|
schedule
|
|
worker_thread
|
|
kthread
|
|
ret_from_fork
|
|
kworker/0:2
|
|
8270305
|
|
|
|
schedule
|
|
rcu_nocb_kthread
|
|
kthread
|
|
ret_from_fork
|
|
rcuos/3
|
|
8349697
|
|
|
|
schedule
|
|
rcu_nocb_kthread
|
|
kthread
|
|
ret_from_fork
|
|
rcuos/2
|
|
8363357
|
|
|
|
schedule
|
|
rcu_nocb_kthread
|
|
kthread
|
|
ret_from_fork
|
|
rcuos/1
|
|
8365338
|
|
|
|
schedule
|
|
schedule_timeout
|
|
xfs_buf_terminate
|
|
kthread
|
|
ret_from_fork
|
|
xfsaild/md0
|
|
8371514
|
|
|
|
schedule
|
|
rcu_nocb_kthread
|
|
kthread
|
|
ret_from_fork
|
|
rcuos/4
|
|
8384013
|
|
|
|
schedule
|
|
rcu_nocb_kthread
|
|
kthread
|
|
ret_from_fork
|
|
rcuos/5
|
|
8390016
|
|
|
|
schedule
|
|
rcu_nocb_kthread
|
|
kthread
|
|
ret_from_fork
|
|
ddebug_tables
|
|
rcuos/0
|
|
8405428
|
|
|
|
schedule
|
|
schedule_timeout
|
|
rcu_gp_kthread
|
|
kthread
|
|
ret_from_fork
|
|
ddebug_tables
|
|
rcu_sched
|
|
8406930
|
|
|
|
schedule
|
|
rcu_nocb_kthread
|
|
kthread
|
|
ret_from_fork
|
|
rcuos/7
|
|
8409575
|
|
|
|
schedule
|
|
rcu_nocb_kthread
|
|
kthread
|
|
ret_from_fork
|
|
rcuos/6
|
|
8415062
|
|
|
|
schedule
|
|
schedule_hrtimeout_range_clock
|
|
schedule_hrtimeout_range
|
|
poll_schedule_timeout
|
|
do_select
|
|
core_sys_select
|
|
sys_select
|
|
entry_SYSCALL_64_fastpath
|
|
offcputime
|
|
8421478
|
|
|
|
schedule
|
|
worker_thread
|
|
kthread
|
|
ret_from_fork
|
|
kworker/4:0
|
|
8421492
|
|
|
|
schedule
|
|
schedule_hrtimeout_range_clock
|
|
schedule_hrtimeout_range
|
|
poll_schedule_timeout
|
|
do_select
|
|
core_sys_select
|
|
sys_select
|
|
entry_SYSCALL_64_fastpath
|
|
sshd
|
|
14249005
|
|
|
|
schedule
|
|
schedule_hrtimeout_range_clock
|
|
schedule_hrtimeout_range
|
|
poll_schedule_timeout
|
|
do_sys_poll
|
|
sys_poll
|
|
entry_SYSCALL_64_fastpath
|
|
supervise
|
|
81670888
|
|
|
|
The last few stack traces aren't very interesting, since they are threads that
|
|
are often blocked off-CPU waiting for work.
|
|
|
|
Do be somewhat careful with overhead: this is tracing scheduler functions, which
|
|
can be called very frequently. While this uses in-kernel summaries for
|
|
efficiency, the rate of scheduler functions can be very high (> 1,000,000/sec),
|
|
and this is performing stack walks when threads return to CPU. At some point
|
|
the overhead will be measurable.
|
|
|
|
|
|
A -p option can be used to filter (in-kernel) on a single process ID. For
|
|
example, only matching PID 26651, which is a running "dd" command:
|
|
|
|
# ./offcputime -K -p 26651
|
|
Tracing off-CPU time (us) of all threads by kernel stack... Hit Ctrl-C to end.
|
|
^C
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
do_blockdev_direct_IO
|
|
__blockdev_direct_IO
|
|
blkdev_direct_IO
|
|
generic_file_read_iter
|
|
blkdev_read_iter
|
|
__vfs_read
|
|
vfs_read
|
|
sys_read
|
|
entry_SYSCALL_64_fastpath
|
|
dd
|
|
2405710
|
|
|
|
The stack trace shows "dd" is blocked waiting on disk I/O, as expected, for a
|
|
total of 2.4 seconds during tracing.
|
|
|
|
|
|
A duration can be added, for example, tracing for 5 seconds only:
|
|
|
|
# ./offcputime -K -p 26651 5
|
|
Tracing off-CPU time (us) of all threads by kernel stack for 5 secs.
|
|
|
|
schedule
|
|
schedule_timeout
|
|
io_schedule_timeout
|
|
do_blockdev_direct_IO
|
|
__blockdev_direct_IO
|
|
blkdev_direct_IO
|
|
generic_file_read_iter
|
|
blkdev_read_iter
|
|
__vfs_read
|
|
vfs_read
|
|
sys_read
|
|
entry_SYSCALL_64_fastpath
|
|
dd
|
|
4413909
|
|
|
|
Here, dd was blocked for 4.4 seconds out of 5. Or put differently, likely
|
|
on-CPU for about 12% of the time. Which matches the ratio seen by time(1):
|
|
|
|
# time dd if=/dev/md0 iflag=direct of=/dev/null bs=1k
|
|
^C108115+0 records in
|
|
108114+0 records out
|
|
110708736 bytes (111 MB) copied, 13.7565 s, 8.0 MB/s
|
|
|
|
real 0m13.760s
|
|
user 0m0.000s
|
|
sys 0m1.739s
|
|
|
|
|
|
A -f option will emit output using the "folded stacks" format, which can be
|
|
read directly by flamegraph.pl from the FlameGraph open source software
|
|
(https://github.com/brendangregg/FlameGraph). Eg:
|
|
|
|
# ./offcputime -K -f 5
|
|
bash;entry_SYSCALL_64_fastpath;sys_read;vfs_read;__vfs_read;tty_read;n_tty_read;call_rwsem_down_read_failed;rwsem_down_read_failed;schedule 8
|
|
yes;entry_SYSCALL_64_fastpath;sys_write;vfs_write;__vfs_write;tty_write;n_tty_write;call_rwsem_down_read_failed;rwsem_down_read_failed;schedule 14
|
|
run;page_fault;do_page_fault;__do_page_fault;handle_mm_fault;__do_fault;filemap_fault;__lock_page_or_retry;wait_on_page_bit_killable;__wait_on_bit;bit_wait_io;io_schedule_timeout;schedule_timeout;schedule 33
|
|
rcuos/4;ret_from_fork;kthread;rcu_nocb_kthread;schedule 45
|
|
bash;entry_SYSCALL_64_fastpath;sys_read;vfs_read;__vfs_read;pipe_read;pipe_wait;schedule 88
|
|
run;page_fault;do_page_fault;__do_page_fault;handle_mm_fault;__do_fault;filemap_fault;__lock_page_or_retry;wait_on_page_bit_killable;__wait_on_bit;bit_wait_io;io_schedule_timeout;schedule_timeout;schedule 108
|
|
jbd2/xvda1-8;mb_cache_list;ret_from_fork;kthread;kjournald2;jbd2_journal_commit_transaction;__wait_on_buffer;out_of_line_wait_on_bit;__wait_on_bit;bit_wait_io;io_schedule_timeout;schedule_timeout;schedule 828
|
|
jbd2/xvda1-8;mb_cache_list;ret_from_fork;kthread;kjournald2;jbd2_journal_commit_transaction;__wait_on_buffer;out_of_line_wait_on_bit;__wait_on_bit;bit_wait_io;io_schedule_timeout;schedule_timeout;schedule 6201
|
|
supervise;entry_SYSCALL_64_fastpath;sys_rename;dput;__dentry_kill;iput;evict;ext4_evict_inode;truncate_inode_pages_final;truncate_inode_pages_range;wait_on_page_bit;__wait_on_bit;bit_wait_io;io_schedule_timeout;schedule_timeout;schedule 41049
|
|
run;entry_SYSCALL_64_fastpath;sys_wait4;do_wait;schedule 120709
|
|
bash;entry_SYSCALL_64_fastpath;sys_wait4;do_wait;schedule 699320
|
|
ksoftirqd/0;ret_from_fork;kthread;smpboot_thread_fn;schedule 1077529
|
|
bash;entry_SYSCALL_64_fastpath;sys_read;vfs_read;__vfs_read;tty_read;n_tty_read;wait_woken;schedule_timeout;schedule 1362045
|
|
sshd;entry_SYSCALL_64_fastpath;sys_select;core_sys_select;do_select;poll_schedule_timeout;schedule_hrtimeout_range;schedule_hrtimeout_range_clock;schedule 1377627
|
|
migration/0;ddebug_tables;ret_from_fork;kthread;smpboot_thread_fn;schedule 2040753
|
|
snmpd;entry_SYSCALL_64_fastpath;sys_select;core_sys_select;do_select;poll_schedule_timeout;schedule_hrtimeout_range;schedule_hrtimeout_range_clock;schedule 2197568
|
|
migration/5;ret_from_fork;kthread;smpboot_thread_fn;schedule 3079426
|
|
migration/7;ret_from_fork;kthread;smpboot_thread_fn;schedule 3084746
|
|
kworker/6:2;ret_from_fork;kthread;worker_thread;schedule 3940583
|
|
kworker/5:1;ret_from_fork;kthread;worker_thread;schedule 3944892
|
|
kworker/1:2;ret_from_fork;kthread;worker_thread;schedule 3999646
|
|
ntpd;entry_SYSCALL_64_fastpath;sys_select;core_sys_select;do_select;poll_schedule_timeout;schedule_hrtimeout_range;schedule_hrtimeout_range_clock;schedule 3999904
|
|
kworker/u16:0;ret_from_fork;kthread;worker_thread;schedule 3999967
|
|
kworker/7:0;ret_from_fork;kthread;worker_thread;schedule 3999987
|
|
tail;entry_SYSCALL_64_fastpath;sys_nanosleep;hrtimer_nanosleep;do_nanosleep;schedule 4000473
|
|
migration/1;ret_from_fork;kthread;smpboot_thread_fn;schedule 4091150
|
|
migration/4;ret_from_fork;kthread;smpboot_thread_fn;schedule 4095217
|
|
readproctitle;entry_SYSCALL_64_fastpath;sys_read;vfs_read;__vfs_read;pipe_read;pipe_wait;schedule 4108470
|
|
migration/3;ret_from_fork;kthread;smpboot_thread_fn;schedule 4109264
|
|
migration/2;ret_from_fork;kthread;smpboot_thread_fn;schedule 4109280
|
|
migration/6;ret_from_fork;kthread;smpboot_thread_fn;schedule 4111143
|
|
kworker/4:0;ret_from_fork;kthread;worker_thread;schedule 4402350
|
|
kworker/3:0;ret_from_fork;kthread;worker_thread;schedule 4433988
|
|
kworker/2:1;ret_from_fork;kthread;worker_thread;schedule 4636142
|
|
kworker/0:2;ret_from_fork;kthread;worker_thread;schedule 4832023
|
|
rcuos/1;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4974186
|
|
rcuos/5;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4977137
|
|
rcuos/6;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4987769
|
|
rcuos/3;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4992282
|
|
rcuos/4;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4992364
|
|
rcuos/2;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4992714
|
|
rcuos/0;ddebug_tables;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4996504
|
|
rcuos/7;ret_from_fork;kthread;rcu_nocb_kthread;schedule 4998497
|
|
rcu_sched;ddebug_tables;ret_from_fork;kthread;rcu_gp_kthread;schedule_timeout;schedule 5000686
|
|
offcputime;entry_SYSCALL_64_fastpath;sys_select;core_sys_select;do_select;poll_schedule_timeout;schedule_hrtimeout_range;schedule_hrtimeout_range_clock;schedule 5005063
|
|
dd;entry_SYSCALL_64_fastpath;sys_read;vfs_read;__vfs_read;blkdev_read_iter;generic_file_read_iter;blkdev_direct_IO;__blockdev_direct_IO;do_blockdev_direct_IO;io_schedule_timeout;schedule_timeout;schedule 8025599
|
|
supervise;entry_SYSCALL_64_fastpath;sys_poll;do_sys_poll;poll_schedule_timeout;schedule_hrtimeout_range;schedule_hrtimeout_range_clock;schedule 40835611
|
|
|
|
The stack traces are shown as single lines, with functions separated by
|
|
semicolons. The first entry is the task name. The 2nd column is the total
|
|
off-CPU time.
|
|
|
|
I'd save this output to a file, then move it to the system where you'll be
|
|
creating your "off-CPU time flame graphs".
|
|
|
|
|
|
USAGE message:
|
|
|
|
# ./offcputime.py -h
|
|
usage: offcputime.py [-h] [-p PID | -t TID | -u | -k] [-U | -K] [-d] [-f]
|
|
[--stack-storage-size STACK_STORAGE_SIZE]
|
|
[-m MIN_BLOCK_TIME] [-M MAX_BLOCK_TIME] [--state STATE]
|
|
[duration]
|
|
|
|
Summarize off-CPU time by stack trace
|
|
|
|
positional arguments:
|
|
duration duration of trace, in seconds
|
|
|
|
optional arguments:
|
|
-h, --help show this help message and exit
|
|
-p PID, --pid PID trace this PID only
|
|
-t TID, --tid TID trace this TID only
|
|
-u, --user-threads-only
|
|
user threads only (no kernel threads)
|
|
-k, --kernel-threads-only
|
|
kernel threads only (no user threads)
|
|
-U, --user-stacks-only
|
|
show stacks from user space only (no kernel space
|
|
stacks)
|
|
-K, --kernel-stacks-only
|
|
show stacks from kernel space only (no user space
|
|
stacks)
|
|
-d, --delimited insert delimiter between kernel/user stacks
|
|
-f, --folded output folded format
|
|
--stack-storage-size STACK_STORAGE_SIZE
|
|
the number of unique stack traces that can be stored
|
|
and displayed (default 1024)
|
|
-m MIN_BLOCK_TIME, --min-block-time MIN_BLOCK_TIME
|
|
the amount of time in microseconds over which we store
|
|
traces (default 1)
|
|
-M MAX_BLOCK_TIME, --max-block-time MAX_BLOCK_TIME
|
|
the amount of time in microseconds under which we
|
|
store traces (default U64_MAX)
|
|
--state STATE filter on this thread state bitmask (eg, 2 ==
|
|
TASK_UNINTERRUPTIBLE) see include/linux/sched.h
|
|
|
|
examples:
|
|
./offcputime # trace off-CPU stack time until Ctrl-C
|
|
./offcputime 5 # trace for 5 seconds only
|
|
./offcputime -f 5 # 5 seconds, and output in folded format
|
|
./offcputime -m 1000 # trace only events that last more than 1000 usec
|
|
./offcputime -M 10000 # trace only events that last less than 10000 usec
|
|
./offcputime -p 185 # only trace threads for PID 185
|
|
./offcputime -t 188 # only trace thread 188
|
|
./offcputime -u # only trace user threads (no kernel)
|
|
./offcputime -k # only trace kernel threads (no user)
|
|
./offcputime -U # only show user space stacks (no kernel)
|
|
./offcputime -K # only show kernel space stacks (no user)
|