Complete report #2060
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc00a93be
Build id: 3.10.0-957.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc00c471e
Build id: 3.10.0-957.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc010c2e0
Build id: 3.10.0-957.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffa1ed0960
Build id: 3.10.0-957.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffa1ed9efc
Build id: 3.10.0-957.el7.x86_64
|
0x4cc
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffa1d42ff9
Build id: 3.10.0-957.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffa1d43076
Build id: 3.10.0-957.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffa1ed84a8
Build id: 3.10.0-957.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffa1eda0f0
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffa1eda245
Build id: 3.10.0-957.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffa1eda4b8
Build id: 3.10.0-957.el7.x86_64
|
0x168
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffffa1ecf96c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffffa1ed9a02
Build id: 3.10.0-957.el7.x86_64
|
0x132
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffffa1d4cce6
Build id: 3.10.0-957.el7.x86_64
|
0x96
|
15
|
__do_softirq
|
vmlinux |
0xffffffffa1aa0f05
Build id: 3.10.0-957.el7.x86_64
|
0xf5
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffffa1aa10c8
Build id: 3.10.0-957.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa1aca5c4
Build id: 3.10.0-957.el7.x86_64
|
0x144
|
18
|
|
vmlinux |
0xffffffffa1aca480
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffffa1ac1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
20
|
|
vmlinux |
0xffffffffa1ac1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
21
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa2174c37
Build id: 3.10.0-957.el7.x86_64
|
0x21
|
22
|
|
vmlinux |
0xffffffffa1ac1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 21s! [ksoftirqd/7:44]
Modules linked in: tcp_diag inet_diag xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device
snd_pcm snd_timer snd joydev pcspkr soundcore sg i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm serio_raw e1000 ata_piix drm_panel_orientation_quirks libata mptspi scsi_transport_spi mptscsih mptbase nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 7 PID: 44 Comm: ksoftirqd/7 Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffff9564f994a080 ti: ffff9564f9960000 task.ti: ffff9564f9960000
RIP: 0010:[<ffffffffc00a93be>] [<ffffffffc00a93be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff9564f9963aa0 EFLAGS: 00000246
RAX: ffffb978c1980000 RBX: ffff956714af5f00 RCX: 000000000000000c
RDX: ffff9567a9359600 RSI: ffff9567a9125000 RDI: 000000000000000e
RBP: ffff9564f9963ab0 R08: 0000000000000003 R09: 0000000282027fff
R10: ffff9567aedf10d8 R11: fffff943c067d500 R12: 0000000100000019
R13: ffff9565b2633900 R14: ffff9564f9963a14 R15: ffffffffa1d4b83b
FS: 0000000000000000(0000) GS:ffff9567ad7c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f980e9eb088 CR3: 00000001f34b6000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc00c471e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc010c2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa1ed0960>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa1ed9efc>] scsi_request_fn+0x4cc/0x680
[<ffffffffa1d42ff9>] __blk_run_queue+0x39/0x50
[<ffffffffa1d43076>] blk_run_queue+0x26/0x40
[<ffffffffa1ed84a8>] scsi_run_queue+0x258/0x2f0
[<ffffffffa1eda0f0>] scsi_next_command+0x20/0x40
[<ffffffffa1eda245>] scsi_end_request+0x135/0x1e0
[<ffffffffa1eda4b8>] scsi_io_completion+0x168/0x6a0
[<ffffffffa1ecf96c>] scsi_finish_command+0xdc/0x140
[<ffffffffa1ed9a02>] scsi_softirq_done+0x132/0x160
[<ffffffffa1d4cce6>] blk_done_softirq+0x96/0xc0
[<ffffffffa1aa0f05>] __do_softirq+0xf5/0x280
[<ffffffffa1aa10c8>] run_ksoftirqd+0x38/0x50
[<ffffffffa1aca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa1aca480>] ? lg_double_unlock+0x40/0x40
[<ffffffffa1ac1c31>] kthread+0xd1/0xe0
[<ffffffffa1ac1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2174c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa1ac1b60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 73 0b
Complete report #1018058
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc030844e
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc036a74e
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03232f0
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb8103160
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb810cd0c
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffb7f68799
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffb7f68826
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffb810b1e8
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffb810cf20
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffb810d08d
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x14d
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffb810d310
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x170
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffffb81020d0
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xe0
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffffb810c828
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x138
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffffb7f729c6
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x96
|
15
|
__do_softirq
|
vmlinux |
0xffffffffb7ca9595
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xf5
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffffb7ca9768
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb7cd43fc
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x14c
|
18
|
|
vmlinux |
0xffffffffb7cd42b0
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffffb7ccb621
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xd1
|
20
|
|
vmlinux |
0xffffffffb7ccb550
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x40
|
21
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb83c51dd
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x7
|
22
|
|
vmlinux |
0xffffffffb7ccb550
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [ksoftirqd/11:67]
Modules linked in: xt_owner xt_REDIRECT nf_nat_redirect xt_multiport ipt_rpfilter iptable_raw ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel rpcsec_gss_krb5 veth auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache xt_set ip_set_hash_ipportnet ip_set_bitmap_port ip_set_hash_ipportip ip_set_hash_ipport dummy nf_tables ip_set ip_vs_sh ip_vs_wrr ip_vs_rr ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_filter ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6_tables xt_mark xt_comment iptable_mangle xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat xt_addrtype iptable_filter scini(POE) ip_vs nf_conntrack binfmt_misc vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon
sg parport_pc joydev parport pcspkr vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix mptspi scsi_transport_spi mptscsih drm libata crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod br_netfilter bridge stp llc overlay(T)
CPU: 11 PID: 67 Comm: ksoftirqd/11 Tainted: P W OEL ------------ T 3.10.0-1160.108.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffffa05f7d559080 ti: ffffa05f7d564000 task.ti: ffffa05f7d564000
RIP: 0010:[<ffffffffc030844e>] [<ffffffffc030844e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
IPVS: rr: TCP 10.104.197.84:9200 - no destination available
PowerFlex netCon_MitEntry_CK:1618 :Mit: ffffa05d750fd718. Chan ffffb2c3c241c010, Con ffffa05d750fd640 ConnectToPeer returned 4(NOT_CONN)
RSP: 0018:ffffa05f7d567aa0 EFLAGS: 00000246
RAX: ffffb2c3c1c00000 RBX: ffffa05f02bde520 RCX: 0000000000000004
RDX: ffffa05df7b58c00 RSI: ffffa05efaffb000 RDI: 000000000000000e
RBP: ffffa05f7d567ab0 R08: 0000000000000003 R09: 0000000226d28fff
R10: 0000000055415f01 R11: ffffa05f55415180 R12: 0000000100000002
R13: ffffa05feb7c2c00 R14: ffffa05f7d567a14 R15: ffffffffb7f7154b
FS: 0000000000000000(0000) GS:ffffa05fffd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005638efe2b000 CR3: 000000003ee10000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc036a74e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03232f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb8103160>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffffb810cd0c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb7f68799>] __blk_run_queue+0x39/0x50
[<ffffffffb7f68826>] blk_run_queue+0x26/0x40
[<ffffffffb810b1e8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb810cf20>] scsi_next_command+0x20/0x40
[<ffffffffb810d08d>] scsi_end_request+0x14d/0x200
[<ffffffffb810d310>] scsi_io_completion+0x170/0x720
[<ffffffffb81020d0>] scsi_finish_command+0xe0/0x140
[<ffffffffb810c828>] scsi_softirq_done+0x138/0x170
[<ffffffffb7f729c6>] blk_done_softirq+0x96/0xd0
[<ffffffffb7ca9595>] __do_softirq+0xf5/0x290
[<ffffffffb7ca9768>] run_ksoftirqd+0x38/0x60
[<ffffffffb7cd43fc>] smpboot_thread_fn+0x14c/0x1b0
[<ffffffffb7cd42b0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb7ccb621>] kthread+0xd1/0xe0
[<ffffffffb7ccb550>] ? insert_kthread_work+0x40/0x40
[<ffffffffb83c51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffb7ccb550>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7
Complete report #1008790
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc02b344e
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc02ce74e
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc02912f0
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb5b03150
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb5b0ccfc
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffb59687b9
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffb5968846
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffb5b0b1d8
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffb5b0cf10
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffb5b0d07d
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x14d
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffb5b0d300
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x170
|
12
|
|
vmlinux |
0xffffffffb56e5d58
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x78
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffffb5b020c0
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xe0
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffffb5b0c818
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x138
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffffb59729e6
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffffb56a9595
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffffb56a9768
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb56d43fc
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x14c
|
19
|
|
vmlinux |
0xffffffffb56d42b0
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffffb56cb621
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffffb56cb550
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb5dc51f7
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffffb56cb550
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ksoftirqd/6:39]
Modules linked in: traps(OE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nf_log_ipv4 nf_log_common xt_LOG ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc squashfs loop sb_edac iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev vmw_balloon sg parport_pc parport vmw_vmci i2c_piix4 binfmt_misc ip_tables
xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit crct10dif_pclmul ata_piix crct10dif_common drm serio_raw crc32c_intel libnvdimm libata mptspi scsi_transport_spi mptscsih mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 6 PID: 39 Comm: ksoftirqd/6 Kdump: loaded Tainted: G OE ------------ 3.10.0-1160.114.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff965299693180 ti: ffff9652996a0000 task.ti: ffff9652996a0000
RIP: 0010:[<ffffffffc02b344e>] [<ffffffffc02b344e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
RSP: 0018:ffff9652996a3aa0 EFLAGS: 00000246
RAX: ffffa8e886ac0000 RBX: ffff9660f778d120 RCX: 000000000000004b
RDX: ffff965a403d1200 RSI: ffff96613a282000 RDI: 000000000000000e
RBP: ffff9652996a3ab0 R08: 0000000000000003 R09: 0000000fcb994fff
R10: ffff96617fda3cf0 R11: fffff646badaad00 R12: 0000000100000002
R13: ffff965fdc598a80 R14: ffff9652996a3a14 R15: ffffffffb597156b
FS: 0000000000000000(0000) GS:ffff96617fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fec76197000 CR3: 0000000ff6dbe000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc02ce74e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02912f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb5b03150>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffffb5b0ccfc>] scsi_request_fn+0x4ac/0x680
[<ffffffffb59687b9>] __blk_run_queue+0x39/0x50
[<ffffffffb5968846>] blk_run_queue+0x26/0x40
[<ffffffffb5b0b1d8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb5b0cf10>] scsi_next_command+0x20/0x40
[<ffffffffb5b0d07d>] scsi_end_request+0x14d/0x200
[<ffffffffb5b0d300>] scsi_io_completion+0x170/0x720
[<ffffffffb56e5d58>] ? __enqueue_entity+0x78/0x80
[<ffffffffb5b020c0>] scsi_finish_command+0xe0/0x140
[<ffffffffb5b0c818>] scsi_softirq_done+0x138/0x170
[<ffffffffb59729e6>] blk_done_softirq+0x96/0xd0
[<ffffffffb56a9595>] __do_softirq+0xf5/0x290
[<ffffffffb56a9768>] run_ksoftirqd+0x38/0x60
[<ffffffffb56d43fc>] smpboot_thread_fn+0x14c/0x1b0
[<ffffffffb56d42b0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb56cb621>] kthread+0xd1/0xe0
[<ffffffffb56cb550>] ? insert_kthread_work+0x40/0x40
[<ffffffffb5dc51f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb56cb550>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7
Complete report #688206
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc03983be
Build id: 3.10.0-957.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc037a71e
Build id: 3.10.0-957.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03c22e0
Build id: 3.10.0-957.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff818d0960
Build id: 3.10.0-957.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff818d9efc
Build id: 3.10.0-957.el7.x86_64
|
0x4cc
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff81742ff9
Build id: 3.10.0-957.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff81743076
Build id: 3.10.0-957.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff818d84a8
Build id: 3.10.0-957.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff818da0f0
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff818da245
Build id: 3.10.0-957.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff818da4b8
Build id: 3.10.0-957.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffff81b678fa
Build id: 3.10.0-957.el7.x86_64
|
0x5ea
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff818cf96c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff818d9a02
Build id: 3.10.0-957.el7.x86_64
|
0x132
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff8174cce6
Build id: 3.10.0-957.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffff814a0f05
Build id: 3.10.0-957.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff814a10c8
Build id: 3.10.0-957.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff814ca5c4
Build id: 3.10.0-957.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffff814ca480
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffff814c1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffff814c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff81b74c24
Build id: 3.10.0-957.el7.x86_64
|
0xe
|
23
|
|
vmlinux |
0xffffffff814c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:3]
Modules linked in: rfcomm bnep btusb btrtl btbcm btintel bluetooth rfkill tcp_lp nls_utf8 isofs fuse ebtable_filter ebtables devlink target_core_user uio target_core_pscsi target_core_file nf_conntrack_netlink target_core_iblock nfnetlink iscsi_target_mod target_core_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi openvswitch nf_nat_ipv6 ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables vmw_vsock_vmci_transport vsock xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_comment xt_conntrack nf_conntrack xt_multiport iptable_filter sunrpc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon
aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 joydev loop pcspkr snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore sg parport_pc parport vmw_vmci i2c_piix4 br_netfilter bridge stp llc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm e1000 mptspi scsi_transport_spi mptscsih drm_panel_orientation_quirks mptbase nfit libnvdimm ata_generic pata_acpi ata_piix libata dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff8f26fcf72080 ti: ffff8f26fc800000 task.ti: ffff8f26fc800000
RIP: 0010:[<ffffffffc03983be>] [<ffffffffc03983be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8f26fc803aa0 EFLAGS: 00000246
RAX: ffffac8940cc0000 RBX: ffff8f26ac0241a0 RCX: 0000000000000003
RDX: ffff8f27554c4400 RSI: ffff8f25b5461000 RDI: 000000000000000e
RBP: ffff8f26fc803ab0 R08: 0000000000000003 R09: 0000000150246fff
R10: ffff8f25b54630d8 R11: fffff58088ffa400 R12: 000000010000000e
R13: ffff8f2589bf4000 R14: ffff8f26fc803a14 R15: ffffffff8174b83b
FS: 0000000000000000(0000) GS:ffff8f27b5600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f76e5a2d000 CR3: 00000001c2f78000 CR4: 00000000003406f0
Call Trace:
[<ffffffffc037a71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03c22e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff818d0960>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff818d9efc>] scsi_request_fn+0x4cc/0x680
[<ffffffff81742ff9>] __blk_run_queue+0x39/0x50
[<ffffffff81743076>] blk_run_queue+0x26/0x40
[<ffffffff818d84a8>] scsi_run_queue+0x258/0x2f0
[<ffffffff818da0f0>] scsi_next_command+0x20/0x40
[<ffffffff818da245>] scsi_end_request+0x135/0x1e0
[<ffffffff818da4b8>] scsi_io_completion+0x168/0x6a0
[<ffffffff81b678fa>] ? __schedule+0x5ea/0x890
[<ffffffff818cf96c>] scsi_finish_command+0xdc/0x140
[<ffffffff818d9a02>] scsi_softirq_done+0x132/0x160
[<ffffffff8174cce6>] blk_done_softirq+0x96/0xc0
[<ffffffff814a0f05>] __do_softirq+0xf5/0x280
[<ffffffff814a10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff814ca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff814ca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff814c1c31>] kthread+0xd1/0xe0
[<ffffffff814c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff81b74c24>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffff814c1b60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 63 3a
Complete report #657707
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01653be
Build id: 3.10.0-1160.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc018071e
Build id: 3.10.0-1160.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc02162e0
Build id: 3.10.0-1160.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffa26e3980
Build id: 3.10.0-1160.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffa26ed0ac
Build id: 3.10.0-1160.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffa25534d9
Build id: 3.10.0-1160.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffa2553556
Build id: 3.10.0-1160.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffa26eb608
Build id: 3.10.0-1160.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffa26ed2c0
Build id: 3.10.0-1160.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffa26ed415
Build id: 3.10.0-1160.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffa26ed688
Build id: 3.10.0-1160.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffffa26ed668
Build id: 3.10.0-1160.el7.x86_64
|
0x148
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffffa26e297c
Build id: 3.10.0-1160.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffffa26ecbd0
Build id: 3.10.0-1160.el7.x86_64
|
0x130
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffffa255d226
Build id: 3.10.0-1160.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffffa22a4b95
Build id: 3.10.0-1160.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffffa22a4d58
Build id: 3.10.0-1160.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa22ce674
Build id: 3.10.0-1160.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffffa22ce530
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffffa22c5c21
Build id: 3.10.0-1160.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffffa22c5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa2993de4
Build id: 3.10.0-1160.el7.x86_64
|
0xe
|
23
|
|
vmlinux |
0xffffffffa22c5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:6]
Modules linked in: xt_nat ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs xt_comment xt_mark nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper snd_timer cryptd snd ppdev soundcore joydev vmw_balloon pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables
xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel nfit libnvdimm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm serio_raw drm ata_piix mptspi e1000 libata scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffffa0a1773b5280 ti: ffffa0a1777f4000 task.ti: ffffa0a1777f4000
RIP: 0010:[<ffffffffc01653be>] [<ffffffffc01653be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffffa0a1777f7aa0 EFLAGS: 00000246
RAX: ffffaec7c0400000 RBX: ffffa0a172878000 RCX: 0000000000000009
RDX: ffffa0a176db3200 RSI: ffffa0a1358f2000 RDI: 000000000000000e
RBP: ffffa0a1777f7ab0 R08: 0000000000000003 R09: 000000001ba66fff
R10: 00000000358bf801 R11: ffffa0a1358bd500 R12: 00000000005b8d80
R13: 00000000777f7a58 R14: ffffffffa22e54ea R15: ffffa0a1777f7a40
FS: 00007ff70bde8a40(0000) GS:ffffa0a17b600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff70b30ad00 CR3: 0000000004dc0000 CR4: 00000000003407f0
Call Trace:
[<ffffffffc018071e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02162e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa26e3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa26ed0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffffa25534d9>] __blk_run_queue+0x39/0x50
[<ffffffffa2553556>] blk_run_queue+0x26/0x40
[<ffffffffa26eb608>] scsi_run_queue+0x258/0x2f0
[<ffffffffa26ed2c0>] scsi_next_command+0x20/0x40
[<ffffffffa26ed415>] scsi_end_request+0x135/0x1e0
[<ffffffffa26ed688>] scsi_io_completion+0x168/0x720
[<ffffffffa26ed668>] ? scsi_io_completion+0x148/0x720
[<ffffffffa26e297c>] scsi_finish_command+0xdc/0x140
[<ffffffffa26ecbd0>] scsi_softirq_done+0x130/0x160
[<ffffffffa255d226>] blk_done_softirq+0x96/0xc0
[<ffffffffa22a4b95>] __do_softirq+0xf5/0x280
[<ffffffffa22a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffa22ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa22ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffffa22c5c21>] kthread+0xd1/0xe0
[<ffffffffa22c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2993de4>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffffa22c5b50>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 33 17
Complete report #166717
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff939883c5
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x15
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc0432632
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4a2
|
3
|
|
vmlinux |
0xffffffff936eb0a0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x170
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff936e2990
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff936ec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff935527a9
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff93552826
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff936ea618
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff936ec2d0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff936ec425
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff936ec698
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffff93853f18
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x18
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff936e198c
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff936ebbe0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x130
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff9355c4f6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffff932a5695
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff932a5858
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff932cf0e4
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffff932cefa0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffff932c6691
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffff932c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff93992d37
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffff932c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink ipmi_devintf ipmi_msghandler vmw_vsock_vmci_transport vsock sunrpc sb_edac ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper sg cryptd vmw_balloon pcspkr joydev vmxnet3 parport_pc parport i2c_piix4 vmw_vmci binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif drm_kms_helper crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops
ttm crct10dif_pclmul crct10dif_common drm crc32c_intel serio_raw ata_piix libata nfit vmw_pvscsi libnvdimm drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G L ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffffa050bb315230 ti: ffffa050bb34c000 task.ti: ffffa050bb34c000
RIP: 0010:[<ffffffff939883c5>] [<ffffffff939883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffffa050bb34fb08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffa050bb34fac8 RCX: ffffa050badbd060
RDX: ffffaf0541b80000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffffa050bb34fb08 R08: 0000000000000000 R09: ffffa051d0a2cb00
R10: ffffa0526ebe5400 R11: ffffa0526eb4a800 R12: 0000000000000020
R13: ffffa04ffab38900 R14: ffffffff936e9d59 R15: ffffa050bb34fa80
FS: 0000000000000000(0000) GS:ffffa0527fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f743e369dc5 CR3: 00000001a7bc0000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc0432632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff936eb0a0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff936e2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff936ec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff935527a9>] __blk_run_queue+0x39/0x50
[<ffffffff93552826>] blk_run_queue+0x26/0x40
[<ffffffff936ea618>] scsi_run_queue+0x258/0x2f0
[<ffffffff936ec2d0>] scsi_next_command+0x20/0x40
[<ffffffff936ec425>] scsi_end_request+0x135/0x1e0
[<ffffffff936ec698>] scsi_io_completion+0x168/0x720
[<ffffffff93853f18>] ? __netif_receive_skb+0x18/0x60
[<ffffffff936e198c>] scsi_finish_command+0xdc/0x140
[<ffffffff936ebbe0>] scsi_softirq_done+0x130/0x160
[<ffffffff9355c4f6>] blk_done_softirq+0x96/0xc0
[<ffffffff932a5695>] __do_softirq+0xf5/0x280
[<ffffffff932a5858>] run_ksoftirqd+0x38/0x50
[<ffffffff932cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff932cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffff932c6691>] kthread+0xd1/0xe0
[<ffffffff932c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff93992d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff932c65c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [CRTC:37:crtc-0] flip_done timed out
Complete report #519270
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc03223be
Build id: 3.10.0-1160.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc03aa71e
Build id: 3.10.0-1160.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03a32e0
Build id: 3.10.0-1160.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff9fce3980
Build id: 3.10.0-1160.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff9fced0ac
Build id: 3.10.0-1160.el7.x86_64
|
0x4ac
|
6
|
|
vmlinux |
0xffffffff9fa26642
Build id: 3.10.0-1160.el7.x86_64
|
0x1e2
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff9fb534d9
Build id: 3.10.0-1160.el7.x86_64
|
0x39
|
8
|
blk_run_queue
|
vmlinux |
0xffffffff9fb53556
Build id: 3.10.0-1160.el7.x86_64
|
0x26
|
9
|
scsi_run_queue
|
vmlinux |
0xffffffff9fceb608
Build id: 3.10.0-1160.el7.x86_64
|
0x258
|
10
|
scsi_next_command
|
vmlinux |
0xffffffff9fced2c0
Build id: 3.10.0-1160.el7.x86_64
|
0x20
|
11
|
scsi_end_request
|
vmlinux |
0xffffffff9fced415
Build id: 3.10.0-1160.el7.x86_64
|
0x135
|
12
|
scsi_io_completion
|
vmlinux |
0xffffffff9fced688
Build id: 3.10.0-1160.el7.x86_64
|
0x168
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff9fce297c
Build id: 3.10.0-1160.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff9fcecbd0
Build id: 3.10.0-1160.el7.x86_64
|
0x130
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff9fb5d226
Build id: 3.10.0-1160.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffff9f8a4b95
Build id: 3.10.0-1160.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff9f8a4d58
Build id: 3.10.0-1160.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff9f8ce674
Build id: 3.10.0-1160.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffff9f8ce530
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffff9f8c5c21
Build id: 3.10.0-1160.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffff9f8c5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff9ff93df7
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffff9f8c5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ksoftirqd/1:14]
Modules linked in: tcp_lp nls_utf8 isofs xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod xfs libcrc32c iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul joydev pcspkr glue_helper vmw_balloon ablk_helper cryptd ppdev sg parport_pc parport snd_seq_midi snd_seq_midi_event
snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore i2c_piix4 vmw_vmci ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix mptspi drm nfit scsi_transport_spi libata mptscsih libnvdimm serio_raw mptbase e1000 drm_panel_orientation_quirks fuse
CPU: 1 PID: 14 Comm: ksoftirqd/1 Tainted: G L ------------ 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffff8e66ffb18000 ti: ffff8e66ffb20000 task.ti: ffff8e66ffb20000
RIP: 0010:[<ffffffffc03223be>] [<ffffffffc03223be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff8e66ffb23aa0 EFLAGS: 00000246
RAX: ffff99b8c0240000 RBX: ffff8e66fc65acc0 RCX: 0000000000000004
RDX: ffff8e66f5ebb400 RSI: ffff8e66f5754000 RDI: 000000000000000e
RBP: ffff8e66ffb23ab0 R08: 0000000000000003 R09: 000000002cc1cfff
R10: ffff8e66fc663c70 R11: ffffd7a980dc6b00 R12: ffffffff9f8d7229
R13: ffff8e66ffb23a30 R14: ffff8e66d6a51080 R15: ffffffff9f8d7202
FS: 0000000000000000(0000) GS:ffff8e66fc640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000405ed0 CR3: 0000000036fc0000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc03aa71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03a32e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9fce3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9fced0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffff9fa26642>] ? kmem_cache_free+0x1e2/0x200
[<ffffffff9fb534d9>] __blk_run_queue+0x39/0x50
[<ffffffff9fb53556>] blk_run_queue+0x26/0x40
[<ffffffff9fceb608>] scsi_run_queue+0x258/0x2f0
[<ffffffff9fced2c0>] scsi_next_command+0x20/0x40
[<ffffffff9fced415>] scsi_end_request+0x135/0x1e0
[<ffffffff9fced688>] scsi_io_completion+0x168/0x720
[<ffffffff9fce297c>] scsi_finish_command+0xdc/0x140
[<ffffffff9fcecbd0>] scsi_softirq_done+0x130/0x160
[<ffffffff9fb5d226>] blk_done_softirq+0x96/0xc0
[<ffffffff9f8a4b95>] __do_softirq+0xf5/0x280
[<ffffffff9f8a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff9f8ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9f8ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffff9f8c5c21>] kthread+0xd1/0xe0
[<ffffffff9f8c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff9ff93df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9f8c5b50>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 03 33
Complete report #541994
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01ad3be
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc023871e
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01a52e0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffaf8e2950
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffaf8ec07c
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x4ac
|
6
|
|
vmlinux |
0xffffffffaf626702
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x1e2
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffaf752769
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x39
|
8
|
blk_run_queue
|
vmlinux |
0xffffffffaf7527e6
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x26
|
9
|
scsi_run_queue
|
vmlinux |
0xffffffffaf8ea5d8
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x258
|
10
|
scsi_next_command
|
vmlinux |
0xffffffffaf8ec290
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x20
|
11
|
scsi_end_request
|
vmlinux |
0xffffffffaf8ec3e5
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x135
|
12
|
scsi_io_completion
|
vmlinux |
0xffffffffaf8ec658
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x168
|
13
|
|
vmlinux |
0xffffffffaf4e69fb
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x8b
|
14
|
scsi_finish_command
|
vmlinux |
0xffffffffaf8e194c
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xdc
|
15
|
scsi_softirq_done
|
vmlinux |
0xffffffffaf8ebba0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x130
|
16
|
blk_done_softirq
|
vmlinux |
0xffffffffaf75c4b6
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x96
|
17
|
__do_softirq
|
vmlinux |
0xffffffffaf4a5695
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xf5
|
18
|
run_ksoftirqd
|
vmlinux |
0xffffffffaf4a5858
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x38
|
19
|
smpboot_thread_fn
|
vmlinux |
0xffffffffaf4cf0e4
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x144
|
20
|
|
vmlinux |
0xffffffffaf4cefa0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x40
|
21
|
kthread
|
vmlinux |
0xffffffffaf4c6691
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xd1
|
22
|
|
vmlinux |
0xffffffffaf4c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x40
|
23
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffafb92d37
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x21
|
24
|
|
vmlinux |
0xffffffffaf4c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 29s! [ksoftirqd/9:54]
Modules linked in: ip6table_mangle ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_filter iptable_mangle ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables xt_comment xt_mark veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_nat_ipv4 nf_nat br_netfilter bridge stp llc binfmt_misc vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon joydev pcspkr sg parport_pc i2c_piix4 parport vmw_vmci overlay(T) nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack ip_tables xfs dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit drm libnvdimm ata_piix mptspi libata scsi_transport_spi mptscsih crct10dif_pclmul crct10dif_common crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ipmi_msghandler]
CPU: 9 PID: 54 Comm: ksoftirqd/9 Tainted: G L ------------ T 3.10.0-1127.18.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/19/2018
task: ffff93eae9735230 ti: ffff93eae9750000 task.ti: ffff93eae9750000
RIP: 0010:[<ffffffffc01ad3be>] [<ffffffffc01ad3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff93eae9753aa0 EFLAGS: 00000246
RAX: ffffaee1c6a40000 RBX: ffff93f4d8a3ee00 RCX: 000000000000000f
RDX: ffff93f95cc92400 RSI: ffff93f979469000 RDI: 000000000000000e
RBP: ffff93eae9753ab0 R08: 0000000000000002 R09: 0000000dc5860fff
R10: ffff93f978bfe000 R11: ffff93eb3a793800 R12: 0000000100000001
R13: ffff93ea91bb0300 R14: ffff93eae9753a14 R15: ffffffffaf75b08b
FS: 0000000000000000(0000) GS:ffff93f97f440000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f126493e000 CR3: 0000000fdc610000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc023871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01a52e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffaf8e2950>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffaf8ec07c>] scsi_request_fn+0x4ac/0x680
[<ffffffffaf626702>] ? kmem_cache_free+0x1e2/0x200
[<ffffffffaf752769>] __blk_run_queue+0x39/0x50
[<ffffffffaf7527e6>] blk_run_queue+0x26/0x40
[<ffffffffaf8ea5d8>] scsi_run_queue+0x258/0x2f0
[<ffffffffaf8ec290>] scsi_next_command+0x20/0x40
[<ffffffffaf8ec3e5>] scsi_end_request+0x135/0x1e0
[<ffffffffaf8ec658>] scsi_io_completion+0x168/0x720
[<ffffffffaf4e69fb>] ? put_prev_entity+0x8b/0x400
[<ffffffffaf8e194c>] scsi_finish_command+0xdc/0x140
[<ffffffffaf8ebba0>] scsi_softirq_done+0x130/0x160
[<ffffffffaf75c4b6>] blk_done_softirq+0x96/0xc0
[<ffffffffaf4a5695>] __do_softirq+0xf5/0x280
[<ffffffffaf4a5858>] run_ksoftirqd+0x38/0x50
[<ffffffffaf4cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffaf4cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffaf4c6691>] kthread+0xd1/0xe0
[<ffffffffaf4c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffafb92d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffaf4c65c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 1b
Complete report #503590
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc04ab3be
Build id: 3.10.0-1062.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc046f71e
Build id: 3.10.0-1062.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc04152e0
Build id: 3.10.0-1062.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffaa0dc890
Build id: 3.10.0-1062.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffaa0e5efc
Build id: 3.10.0-1062.el7.x86_64
|
0x4cc
|
6
|
|
vmlinux |
0xffffffffa9cd9df0
Build id: 3.10.0-1062.el7.x86_64
|
0x190
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffa9f4cf59
Build id: 3.10.0-1062.el7.x86_64
|
0x39
|
8
|
blk_run_queue
|
vmlinux |
0xffffffffa9f4cfd6
Build id: 3.10.0-1062.el7.x86_64
|
0x26
|
9
|
scsi_run_queue
|
vmlinux |
0xffffffffaa0e44a8
Build id: 3.10.0-1062.el7.x86_64
|
0x258
|
10
|
scsi_next_command
|
vmlinux |
0xffffffffaa0e60f0
Build id: 3.10.0-1062.el7.x86_64
|
0x20
|
11
|
scsi_end_request
|
vmlinux |
0xffffffffaa0e6245
Build id: 3.10.0-1062.el7.x86_64
|
0x135
|
12
|
scsi_io_completion
|
vmlinux |
0xffffffffaa0e6476
Build id: 3.10.0-1062.el7.x86_64
|
0x126
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffffaa0db89c
Build id: 3.10.0-1062.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffffaa0e5a02
Build id: 3.10.0-1062.el7.x86_64
|
0x132
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffffa9f56c56
Build id: 3.10.0-1062.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffffa9ca41e5
Build id: 3.10.0-1062.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffffa9ca43a8
Build id: 3.10.0-1062.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa9ccdb04
Build id: 3.10.0-1062.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffffa9ccd9c0
Build id: 3.10.0-1062.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffffa9cc50d1
Build id: 3.10.0-1062.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffffa9cc5000
Build id: 3.10.0-1062.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffaa38bd37
Build id: 3.10.0-1062.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffffa9cc5000
Build id: 3.10.0-1062.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: xt_recent veth nf_tables xt_statistic ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 rpcsec_gss_krb5 nfsv4 dns_resolver nfs lockd grace fscache nf_conntrack_netlink xt_nat xt_mark xt_addrtype xt_comment ip_set nfnetlink ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio sb_edac iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg joydev parport_pc i2c_piix4 parport
vmw_vmci br_netfilter bridge auth_rpcgss stp sunrpc llc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi ata_piix scsi_transport_spi mptscsih libata crct10dif_pclmul mptbase crct10dif_common crc32c_intel serio_raw vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G L ------------ 3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff97c5fa28d230 ti: ffff97c5fa2f8000 task.ti: ffff97c5fa2f8000
RIP: 0010:[<ffffffffc04ab3be>] [<ffffffffc04ab3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff97c5fa2fbaa0 EFLAGS: 00000246
RAX: ffffbc2d83600000 RBX: ffff97c7c9e74900 RCX: 000000000000001f
RDX: ffff97ccace53c00 RSI: ffff97c8adbb7000 RDI: 000000000000000e
RBP: ffff97c5fa2fbab0 R08: 0000000000000002 R09: 000000026343ffff
R10: ffff97c5f9d3c0d8 R11: ffff97c843f46bc0 R12: 0000000100000001
R13: ffff97c6737bad00 R14: ffff97c5fa2fba14 R15: ffffffffa9f557ab
FS: 0000000000000000(0000) GS:ffff97c8afc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000560a0a27e010 CR3: 0000000145410000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc046f71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04152e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffaa0dc890>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffaa0e5efc>] scsi_request_fn+0x4cc/0x680
[<ffffffffa9cd9df0>] ? try_to_wake_up+0x190/0x390
[<ffffffffa9f4cf59>] __blk_run_queue+0x39/0x50
[<ffffffffa9f4cfd6>] blk_run_queue+0x26/0x40
[<ffffffffaa0e44a8>] scsi_run_queue+0x258/0x2f0
[<ffffffffaa0e60f0>] scsi_next_command+0x20/0x40
[<ffffffffaa0e6245>] scsi_end_request+0x135/0x1e0
[<ffffffffaa0e6476>] scsi_io_completion+0x126/0x6a0
[<ffffffffaa0db89c>] scsi_finish_command+0xdc/0x140
[<ffffffffaa0e5a02>] scsi_softirq_done+0x132/0x160
[<ffffffffa9f56c56>] blk_done_softirq+0x96/0xc0
[<ffffffffa9ca41e5>] __do_softirq+0xf5/0x280
[<ffffffffa9ca43a8>] run_ksoftirqd+0x38/0x50
[<ffffffffa9ccdb04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa9ccd9c0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa9cc50d1>] kthread+0xd1/0xe0
[<ffffffffa9cc5000>] ? insert_kthread_work+0x40/0x40
[<ffffffffaa38bd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa9cc5000>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 93 4b
Complete report #425550
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8d71b2c5
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x15
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc02cd622
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x4a2
|
3
|
|
vmlinux |
0xffffffff8d4a8500
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x170
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8d4a0010
Build id: 3.10.0-862.11.6.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8d4a957c
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x4cc
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff8d31d4e9
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff8d31d566
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff8d4a7b18
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff8d4a9770
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff8d4a98c5
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff8d4a9b38
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x168
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffff8d49f01c
Build id: 3.10.0-862.11.6.el7.x86_64
|
0xdc
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffff8d4a9082
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x132
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffff8d3271a6
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x96
|
15
|
__do_softirq
|
vmlinux |
0xffffffff8d09dba5
Build id: 3.10.0-862.11.6.el7.x86_64
|
0xf5
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffff8d09dd68
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffff8d0c6744
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x144
|
18
|
|
vmlinux |
0xffffffff8d0c6600
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffff8d0bdf21
Build id: 3.10.0-862.11.6.el7.x86_64
|
0xd1
|
20
|
|
vmlinux |
0xffffffff8d0bde50
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x40
|
21
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8d7255dd
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x7
|
22
|
|
vmlinux |
0xffffffff8d0bde50
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 26s! [ksoftirqd/0:3]
Modules linked in: veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter binfmt_misc gsch(OE) redirfs(OE) event_collector_1_9_8557(POE) ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon
ppdev pcspkr sg joydev parport_pc shpchp parport i2c_piix4 vmw_vmci auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata crct10dif_pclmul crct10dif_common i2c_core crc32c_intel serio_raw vmxnet3 vmw_pvscsi dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: P W OEL ------------ 3.10.0-862.11.6.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9755ecc80000 ti: ffff9755ecc88000 task.ti: ffff9755ecc88000
RIP: 0010:[<ffffffff8d71b2c5>] [<ffffffff8d71b2c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9755ecc8bb08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff9755ecc8bac8 RCX: ffff9763cee70060
RDX: ffffa4044ce20000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff9755ecc8bb08 R08: 0000000000000000 R09: ffff9760b0f7e000
R10: ffff97610b09e8d8 R11: ffff97610b041800 R12: 0000000000000020
R13: ffff976cd7a9ef00 R14: ffffffff8d4a7259 R15: ffff9755ecc8ba80
FS: 0000000000000000(0000) GS:ffff97610d800000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000448e20 CR3: 0000000f4e80e000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc02cd622>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8d4a8500>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8d4a0010>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8d4a957c>] scsi_request_fn+0x4cc/0x680
[<ffffffff8d31d4e9>] __blk_run_queue+0x39/0x50
[<ffffffff8d31d566>] blk_run_queue+0x26/0x40
[<ffffffff8d4a7b18>] scsi_run_queue+0x258/0x2f0
[<ffffffff8d4a9770>] scsi_next_command+0x20/0x40
[<ffffffff8d4a98c5>] scsi_end_request+0x135/0x1e0
[<ffffffff8d4a9b38>] scsi_io_completion+0x168/0x6a0
[<ffffffff8d49f01c>] scsi_finish_command+0xdc/0x140
[<ffffffff8d4a9082>] scsi_softirq_done+0x132/0x160
[<ffffffff8d3271a6>] blk_done_softirq+0x96/0xc0
[<ffffffff8d09dba5>] __do_softirq+0xf5/0x280
[<ffffffff8d09dd68>] run_ksoftirqd+0x38/0x50
[<ffffffff8d0c6744>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8d0c6600>] ? lg_double_unlock+0x40/0x40
[<ffffffff8d0bdf21>] kthread+0xd1/0xe0
[<ffffffff8d0bde50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8d7255dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff8d0bde50>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #412189
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01583be
Build id: 3.10.0-1160.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc01c771e
Build id: 3.10.0-1160.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc02762e0
Build id: 3.10.0-1160.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb12e3980
Build id: 3.10.0-1160.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb12ed0ac
Build id: 3.10.0-1160.el7.x86_64
|
0x4ac
|
6
|
|
vmlinux |
0xffffffffb1026642
Build id: 3.10.0-1160.el7.x86_64
|
0x1e2
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffb11534d9
Build id: 3.10.0-1160.el7.x86_64
|
0x39
|
8
|
blk_run_queue
|
vmlinux |
0xffffffffb1153556
Build id: 3.10.0-1160.el7.x86_64
|
0x26
|
9
|
scsi_run_queue
|
vmlinux |
0xffffffffb12eb608
Build id: 3.10.0-1160.el7.x86_64
|
0x258
|
10
|
scsi_next_command
|
vmlinux |
0xffffffffb12ed2c0
Build id: 3.10.0-1160.el7.x86_64
|
0x20
|
11
|
scsi_end_request
|
vmlinux |
0xffffffffb12ed415
Build id: 3.10.0-1160.el7.x86_64
|
0x135
|
12
|
scsi_io_completion
|
vmlinux |
0xffffffffb12ed688
Build id: 3.10.0-1160.el7.x86_64
|
0x168
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffffb12e297c
Build id: 3.10.0-1160.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffffb12ecbd0
Build id: 3.10.0-1160.el7.x86_64
|
0x130
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffffb115d226
Build id: 3.10.0-1160.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffffb0ea4b95
Build id: 3.10.0-1160.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffffb0ea4d58
Build id: 3.10.0-1160.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb0ece674
Build id: 3.10.0-1160.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffffb0ece530
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffffb0ec5c21
Build id: 3.10.0-1160.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffffb0ec5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb1593df7
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffffb0ec5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 21s! [ksoftirqd/3:24]
Modules linked in: veth br_netfilter tun devlink overlay(T) bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec iosf_mbi crc32_pclmul ac97_bus ghash_clmulni_intel snd_seq snd_seq_device aesni_intel snd_pcm lrw gf128mul ppdev glue_helper ablk_helper cryptd snd_timer vmw_balloon snd soundcore joydev sg pcspkr i2c_piix4 vmw_vmci parport_pc parport xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common syscopyarea sysfillrect crc32c_intel sysimgblt fb_sys_fops ttm nfit libnvdimm drm serio_raw mptspi e1000 scsi_transport_spi ata_piix mptscsih libata mptbase drm_panel_orientation_quirks
fuse [last unloaded: ip_tables]
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G B ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9c85b2fdb180 ti: ffff9c85b2fe8000 task.ti: ffff9c85b2fe8000
RIP: 0010:[<ffffffffc01583be>] [<ffffffffc01583be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9c85b2febaa0 EFLAGS: 00000246
RAX: ffffaec4006c0000 RBX: ffff9c8489cedfe0 RCX: 0000000000000003
RDX: ffff9c85ac6d8800 RSI: ffff9c85ac454000 RDI: 000000000000000e
RBP: ffff9c85b2febab0 R08: 0000000000000003 R09: 00000000a650ffff
R10: 0000000087986d01 R11: ffff9c8587986bc0 R12: 0000000100000080
R13: ffff9c84832cd380 R14: ffff9c85b2feba14 R15: ffffffffb115bdfb
FS: 0000000000000000(0000) GS:ffff9c85b96c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffc4d9aa5c8 CR3: 000000012c1cc000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc01c771e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02762e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb12e3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb12ed0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffffb1026642>] ? kmem_cache_free+0x1e2/0x200
[<ffffffffb11534d9>] __blk_run_queue+0x39/0x50
[<ffffffffb1153556>] blk_run_queue+0x26/0x40
[<ffffffffb12eb608>] scsi_run_queue+0x258/0x2f0
[<ffffffffb12ed2c0>] scsi_next_command+0x20/0x40
[<ffffffffb12ed415>] scsi_end_request+0x135/0x1e0
[<ffffffffb12ed688>] scsi_io_completion+0x168/0x720
[<ffffffffb12e297c>] scsi_finish_command+0xdc/0x140
[<ffffffffb12ecbd0>] scsi_softirq_done+0x130/0x160
[<ffffffffb115d226>] blk_done_softirq+0x96/0xc0
[<ffffffffb0ea4b95>] __do_softirq+0xf5/0x280
[<ffffffffb0ea4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffb0ece674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb0ece530>] ? lg_double_unlock+0x40/0x40
[<ffffffffb0ec5c21>] kthread+0xd1/0xe0
[<ffffffffb0ec5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffb1593df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb0ec5b50>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 63 16
Complete report #409859
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01313be
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc016871e
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc014c2e0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff90ae4640
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff90aedd6c
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff90953cf9
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff90953d76
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff90aec2c8
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff90aedf80
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff90aee0d5
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff90aee348
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffff90c59b18
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x18
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff90ae363c
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff90aed890
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x130
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff9095db16
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffff906a4d85
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff906a4f48
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff906cea04
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffff906ce8c0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffff906c5fb1
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffff906c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff90d99df7
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffff906c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ksoftirqd/2:19]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_13402(E) xt_nat veth ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter binfmt_misc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop overlay ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel
lrw ppdev gf128mul vmw_balloon glue_helper ablk_helper cryptd vmw_vmci pcspkr joydev sg i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul libata crct10dif_common nfit libnvdimm crc32c_intel mptspi serio_raw scsi_transport_spi mptscsih vmxnet3 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: P EL ------------ 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffffa0c175c18000 ti: ffffa0c175c14000 task.ti: ffffa0c175c14000
RIP: 0010:[<ffffffffc01313be>] [<ffffffffc01313be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffffa0c175c17aa0 EFLAGS: 00000246
RAX: ffffb13a81c40000 RBX: ffffa0c175c17a48 RCX: 000000000000000f
RDX: ffffa0c036603400 RSI: ffffa0c2eb7d1000 RDI: 000000000000000e
RBP: ffffa0c175c17ab0 R08: 0000000000000003 R09: 00000001219d0fff
R10: ffffa0c43fd23c50 R11: ffffd78649a1c800 R12: ffffffff906d7572
R13: ffffa0c175c17a20 R14: 0000000000000002 R15: ffffa0c43fd1acc0
FS: 0000000000000000(0000) GS:ffffa0c43fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000369f24986000 CR3: 00000003e27e0000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc016871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc014c2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff90ae4640>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff90aedd6c>] scsi_request_fn+0x4ac/0x680
[<ffffffff90953cf9>] __blk_run_queue+0x39/0x50
[<ffffffff90953d76>] blk_run_queue+0x26/0x40
[<ffffffff90aec2c8>] scsi_run_queue+0x258/0x2f0
[<ffffffff90aedf80>] scsi_next_command+0x20/0x40
[<ffffffff90aee0d5>] scsi_end_request+0x135/0x1e0
[<ffffffff90aee348>] scsi_io_completion+0x168/0x720
[<ffffffff90c59b18>] ? __netif_receive_skb+0x18/0x60
[<ffffffff90ae363c>] scsi_finish_command+0xdc/0x140
[<ffffffff90aed890>] scsi_softirq_done+0x130/0x160
[<ffffffff9095db16>] blk_done_softirq+0x96/0xc0
[<ffffffff906a4d85>] __do_softirq+0xf5/0x280
[<ffffffff906a4f48>] run_ksoftirqd+0x38/0x50
[<ffffffff906cea04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff906ce8c0>] ? lg_double_unlock+0x40/0x40
[<ffffffff906c5fb1>] kthread+0xd1/0xe0
[<ffffffff906c5ee0>] ? insert_kthread_work+0x40/0x40
[<ffffffff90d99df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff906c5ee0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 f3 13
Complete report #406509
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb398e9f5
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x15
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc0317632
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x4a2
|
3
|
|
vmlinux |
0xffffffffb36ecd50
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x170
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb36e4640
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb36edd6c
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffb3553cf9
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffb3553d76
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffb36ec2c8
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffb36edf80
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffb36ee0d5
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffb36ee348
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x168
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffffb36e363c
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xdc
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffffb36ed890
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x130
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffffb355db16
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x96
|
15
|
__do_softirq
|
vmlinux |
0xffffffffb32a4d85
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xf5
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffffb32a4f48
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb32cea04
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x144
|
18
|
|
vmlinux |
0xffffffffb32ce8c0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffffb32c5fb1
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xd1
|
20
|
|
vmlinux |
0xffffffffb32c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x40
|
21
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb3999df7
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x21
|
22
|
|
vmlinux |
0xffffffffb32c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [ksoftirqd/1:14]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_13109(E) tcp_lp falcon_lsm_pinned_12905(E) nfsv3 nfs_acl nfs lockd grace fscache xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd pcspkr joydev sg parport_pc vmw_vmci parport i2c_piix4 binfmt_misc auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic
drm_kms_helper ahci crct10dif_pclmul crct10dif_common syscopyarea sysfillrect ata_piix libahci sysimgblt fb_sys_fops crc32c_intel ttm serio_raw libata drm vmxnet3 nfit libnvdimm vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: falcon_kal]
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: P EL ------------ 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff918c38550000 ti: ffff918c38558000 task.ti: ffff918c38558000
RIP: 0010:[<ffffffffb398e9f5>] [<ffffffffb398e9f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff918c3855bb08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff918c3855bac8 RCX: ffff918c38139060
RDX: ffffb6c540780000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff918c3855bb08 R08: 0000000000000000 R09: ffff918bb22aea00
R10: ffff918b34815c00 R11: ffff918c33752800 R12: 0000000000000020
R13: ffff918c0ea39e00 R14: ffffffffb36eba09 R15: ffff918c3855ba80
FS: 0000000000000000(0000) GS:ffff918c3fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6ca63da000 CR3: 000000008b086000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc0317632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffb36ecd50>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffb36e4640>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb36edd6c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb3553cf9>] __blk_run_queue+0x39/0x50
[<ffffffffb3553d76>] blk_run_queue+0x26/0x40
[<ffffffffb36ec2c8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb36edf80>] scsi_next_command+0x20/0x40
[<ffffffffb36ee0d5>] scsi_end_request+0x135/0x1e0
[<ffffffffb36ee348>] scsi_io_completion+0x168/0x720
[<ffffffffb36e363c>] scsi_finish_command+0xdc/0x140
[<ffffffffb36ed890>] scsi_softirq_done+0x130/0x160
[<ffffffffb355db16>] blk_done_softirq+0x96/0xc0
[<ffffffffb32a4d85>] __do_softirq+0xf5/0x280
[<ffffffffb32a4f48>] run_ksoftirqd+0x38/0x50
[<ffffffffb32cea04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb32ce8c0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb32c5fb1>] kthread+0xd1/0xe0
[<ffffffffb32c5ee0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb3999df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb32c5ee0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #404016
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8678ea05
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x15
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc01a5632
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x4a2
|
3
|
|
vmlinux |
0xffffffff864ecf80
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x170
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff864e4870
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff864edf9c
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff86353f29
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff86353fa6
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff864ec4f8
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff864ee1b0
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff864ee305
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff864ee578
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffff8607dfe9
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x109
|
13
|
|
vmlinux |
0xffffffff8607dff5
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x115
|
14
|
scsi_finish_command
|
vmlinux |
0xffffffff864e386c
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xdc
|
15
|
scsi_softirq_done
|
vmlinux |
0xffffffff864edac0
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x130
|
16
|
blk_done_softirq
|
vmlinux |
0xffffffff8635dd46
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x96
|
17
|
__do_softirq
|
vmlinux |
0xffffffff860a4bf5
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xf5
|
18
|
run_ksoftirqd
|
vmlinux |
0xffffffff860a4db8
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x38
|
19
|
smpboot_thread_fn
|
vmlinux |
0xffffffff860ce8b4
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x144
|
20
|
|
vmlinux |
0xffffffff860ce770
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x40
|
21
|
kthread
|
vmlinux |
0xffffffff860c5e61
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xd1
|
22
|
|
vmlinux |
0xffffffff860c5d90
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x40
|
23
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff86799df7
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x21
|
24
|
|
vmlinux |
0xffffffff860c5d90
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink ipmi_devintf ipmi_msghandler vmw_vsock_vmci_transport vsock sunrpc iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon sg joydev pcspkr parport_pc parport i2c_piix4 vmw_vmci binfmt_misc ip_tables xfs libcrc32c ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common
e1000e drm crc32c_intel serio_raw ahci ptp pps_core libahci vmw_pvscsi ata_piix drm_panel_orientation_quirks nfit libata libnvdimm dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G L ------------ 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/28/2017
task: ffff940b76315280 ti: ffff940b7634c000 task.ti: ffff940b7634c000
RIP: 0010:[<ffffffff8678ea05>] [<ffffffff8678ea05>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff940b7634fb08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff940b7634fac8 RCX: ffff940ba6bc9060
RDX: ffffb37fc1b80000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff940b7634fb08 R08: 0000000000000000 R09: ffff940a90995c00
R10: ffff940ba6582000 R11: ffff940ba660f000 R12: 0000000000000020
R13: ffff940d301d5500 R14: ffffffff864ebc39 R15: ffff940b7634fa80
FS: 0000000000000000(0000) GS:ffff940e3fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000006dee20 CR3: 00000003f2c94000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc01a5632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff864ecf80>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff864e4870>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff864edf9c>] scsi_request_fn+0x4ac/0x680
[<ffffffff86353f29>] __blk_run_queue+0x39/0x50
[<ffffffff86353fa6>] blk_run_queue+0x26/0x40
[<ffffffff864ec4f8>] scsi_run_queue+0x258/0x2f0
[<ffffffff864ee1b0>] scsi_next_command+0x20/0x40
[<ffffffff864ee305>] scsi_end_request+0x135/0x1e0
[<ffffffff864ee578>] scsi_io_completion+0x168/0x720
[<ffffffff8607dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffff8607dff5>] ? switch_mm_irqs_off+0x115/0x290
[<ffffffff864e386c>] scsi_finish_command+0xdc/0x140
[<ffffffff864edac0>] scsi_softirq_done+0x130/0x160
[<ffffffff8635dd46>] blk_done_softirq+0x96/0xc0
[<ffffffff860a4bf5>] __do_softirq+0xf5/0x280
[<ffffffff860a4db8>] run_ksoftirqd+0x38/0x50
[<ffffffff860ce8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff860ce770>] ? lg_double_unlock+0x40/0x40
[<ffffffff860c5e61>] kthread+0xd1/0xe0
[<ffffffff860c5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffff86799df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff860c5d90>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #393313
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff816adce5
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x15
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc001b632
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x442
|
3
|
|
vmlinux |
0xffffffff8146f5b0
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x170
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8146782a
Build id: 3.10.0-693.17.1.el7.x86_64
|
0xaa
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff814705ef
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x4df
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff812f96a3
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x33
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff812f9716
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff8146eb08
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff814707d0
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff81470925
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff81470d07
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x2d7
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffff81466845
Build id: 3.10.0-693.17.1.el7.x86_64
|
0xd5
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffff814700e2
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x132
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffff813029e0
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x90
|
15
|
__do_softirq
|
vmlinux |
0xffffffff8109275f
Build id: 3.10.0-693.17.1.el7.x86_64
|
0xef
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffff81092928
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810bae1f
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x12f
|
18
|
|
vmlinux |
0xffffffff810bacf0
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffff810b270f
Build id: 3.10.0-693.17.1.el7.x86_64
|
0xcf
|
20
|
|
vmlinux |
0xffffffff810b2640
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x40
|
21
|
ret_from_fork
|
vmlinux |
0xffffffff816b8798
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x58
|
22
|
|
vmlinux |
0xffffffff810b2640
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#12 stuck for 54s! [ksoftirqd/12:68]
Modules linked in: vfat fat isofs xfs libcrc32c binfmt_misc bmhook(OE) tmhook(OE) dsa_filter(POE) dsa_filter_hook(OE) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod btrfs sb_edac edac_core iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd raid6_pq xor joydev pcspkr sg nfit parport_pc parport libnvdimm vmw_vmci shpchp i2c_piix4 ip_tables ext4 mbcache jbd2 sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ahci ata_piix libahci libata serio_raw vmxnet3 i2c_core vmw_pvscsi floppy
CPU: 12 PID: 68 Comm: ksoftirqd/12 Tainted: P OEL ------------ 3.10.0-693.17.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff880176a89fa0 ti: ffff880176a98000 task.ti: ffff880176a98000
RIP: 0010:[<ffffffff816adce5>] [<ffffffff816adce5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff880176a9bb08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff880176a9bac8 RCX: 00000005a248efff
RDX: ffffc90003190000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff880176a9bb08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8805a15d6d80 R11: ffff8805a4cf5800 R12: 0000000000000020
R13: ffff88019a524d80 R14: ffffffff8146e1d9 R15: ffff880176a9ba80
FS: 0000000000000000(0000) GS:ffff8805a7900000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f83bffa3000 CR3: 00000000019fa000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc001b632>] pvscsi_queue+0x442/0x7e1 [vmw_pvscsi]
[<ffffffff8146f5b0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8146782a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff814705ef>] scsi_request_fn+0x4df/0x680
[<ffffffff812f96a3>] __blk_run_queue+0x33/0x40
[<ffffffff812f9716>] blk_run_queue+0x26/0x40
[<ffffffff8146eb08>] scsi_run_queue+0x258/0x2f0
[<ffffffff814707d0>] scsi_next_command+0x20/0x40
[<ffffffff81470925>] scsi_end_request+0x135/0x1e0
[<ffffffff81470d07>] scsi_io_completion+0x2d7/0x6a0
[<ffffffff81466845>] scsi_finish_command+0xd5/0x130
[<ffffffff814700e2>] scsi_softirq_done+0x132/0x160
[<ffffffff813029e0>] blk_done_softirq+0x90/0xc0
[<ffffffff8109275f>] __do_softirq+0xef/0x280
[<ffffffff81092928>] run_ksoftirqd+0x38/0x50
[<ffffffff810bae1f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810bacf0>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b270f>] kthread+0xcf/0xe0
[<ffffffff810b2640>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b8798>] ret_from_fork+0x58/0x90
[<ffffffff810b2640>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #388785
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff9e3893e5
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x15
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc0080632
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x4a2
|
3
|
|
vmlinux |
0xffffffff9e0ec2a0
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x170
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff9e0e3b90
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff9e0ed2bc
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff9df536d9
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff9df53756
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff9e0eb818
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff9e0ed4d0
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff9e0ed625
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff9e0ed898
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffff9e254b28
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x18
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff9e0e2b8c
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff9e0ecde0
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x130
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff9df5d426
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffff9dca4b95
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff9dca4d58
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff9dcce8c4
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffff9dcce780
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffff9dcc5e71
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffff9dcc5da0
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff9e393ddd
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x7
|
23
|
|
vmlinux |
0xffffffff9dcc5da0
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [ksoftirqd/4:29]
Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache xt_set nf_tables ip6t_MASQUERADE nf_nat_masquerade_ipv6 xt_comment xt_mark ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack_netlink xt_addrtype xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables veth vxlan ip6_udp_tunnel udp_tunnel ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set_bitmap_port
dummy br_netfilter tun devlink overlay(T) binfmt_misc bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg joydev pcspkr vmw_vmci vmw_balloon parport_pc i2c_piix4 parport xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci nfit libnvdimm sd_mod libahci ata_piix crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common libata crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: ip_tables]
CPU: 4 PID: 29 Comm: ksoftirqd/4 Kdump: loaded Tainted: G W L ------------ T 3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff92f9fe758000 ti: ffff92f9fe760000 task.ti: ffff92f9fe760000
RIP: 0010:[<ffffffff9e3893e5>] [<ffffffff9e3893e5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff92f9fe763b08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff92f9fe763ac8 RCX: ffffffff9e834f40
RDX: ffffb56e00ea0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff92f9fe763b08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff92f9efbafc00 R11: ffff92fab469d800 R12: 0000000000000020
R13: ffff92fa3eeab080 R14: ffffffff9e0eaf59 R15: ffff92f9fe763a80
FS: 0000000000000000(0000) GS:ffff92fabfd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000562ee072250f CR3: 0000000042410000 CR4: 00000000007607e0
PKRU: 00000000
Call Trace:
[<ffffffffc0080632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff9e0ec2a0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff9e0e3b90>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9e0ed2bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff9df536d9>] __blk_run_queue+0x39/0x50
[<ffffffff9df53756>] blk_run_queue+0x26/0x40
[<ffffffff9e0eb818>] scsi_run_queue+0x258/0x2f0
[<ffffffff9e0ed4d0>] scsi_next_command+0x20/0x40
[<ffffffff9e0ed625>] scsi_end_request+0x135/0x1e0
[<ffffffff9e0ed898>] scsi_io_completion+0x168/0x720
[<ffffffff9e254b28>] ? __netif_receive_skb+0x18/0x60
[<ffffffff9e0e2b8c>] scsi_finish_command+0xdc/0x140
[<ffffffff9e0ecde0>] scsi_softirq_done+0x130/0x160
[<ffffffff9df5d426>] blk_done_softirq+0x96/0xc0
[<ffffffff9dca4b95>] __do_softirq+0xf5/0x280
[<ffffffff9dca4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff9dcce8c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9dcce780>] ? lg_double_unlock+0x40/0x40
[<ffffffff9dcc5e71>] kthread+0xd1/0xe0
[<ffffffff9dcc5da0>] ? insert_kthread_work+0x40/0x40
[<ffffffff9e393ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff9dcc5da0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #353790
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc04453be
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc04bb71e
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc04282e0
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff974e4860
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff974edf8c
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff97353f39
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff97353fb6
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff974ec4e8
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff974ee1a0
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff974ee2f5
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff974ee568
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffff970dfbd8
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x78
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff974e385c
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff974edab0
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x130
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff9735dd36
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffff970a4bf5
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff970a4db8
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff970ce8b4
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffff970ce770
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffff970c5e61
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffff970c5d90
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff97795df7
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffff970c5d90
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#30 stuck for 22s! [ksoftirqd/30:163]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon pcspkr sg joydev parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi scsi_transport_spi mptscsih libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks
floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 30 PID: 163 Comm: ksoftirqd/30 Kdump: loaded Tainted: G L ------------ 3.10.0-1160.53.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff99e01e79e300 ti: ffff99e01e7d0000 task.ti: ffff99e01e7d0000
RIP: 0010:[<ffffffffc04453be>] [<ffffffffc04453be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff99e01e7d3aa0 EFLAGS: 00000246
RAX: ffffa7c841c40000 RBX: ffff99e03fde9100 RCX: 0000000000000009
RDX: ffff99df9c368000 RSI: ffff99def9e50000 RDI: 000000000000000e
RBP: ffff99e01e7d3ab0 R08: 0000000000000002 R09: 0000000016739fff
R10: ffff99e03b753c00 R11: ffff99e03b4c1c00 R12: 0000000100000001
R13: ffff99ddcbbc4000 R14: ffff99e01e7d3a14 R15: ffffffff9735c8fb
FS: 0000000000000000(0000) GS:ffff99e03fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd1ba14e7a0 CR3: 00000001e7610000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc04bb71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04282e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff974e4860>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff974edf8c>] scsi_request_fn+0x4ac/0x680
[<ffffffff97353f39>] __blk_run_queue+0x39/0x50
[<ffffffff97353fb6>] blk_run_queue+0x26/0x40
[<ffffffff974ec4e8>] scsi_run_queue+0x258/0x2f0
[<ffffffff974ee1a0>] scsi_next_command+0x20/0x40
[<ffffffff974ee2f5>] scsi_end_request+0x135/0x1e0
[<ffffffff974ee568>] scsi_io_completion+0x168/0x720
[<ffffffff970dfbd8>] ? __enqueue_entity+0x78/0x80
[<ffffffff974e385c>] scsi_finish_command+0xdc/0x140
[<ffffffff974edab0>] scsi_softirq_done+0x130/0x160
[<ffffffff9735dd36>] blk_done_softirq+0x96/0xc0
[<ffffffff970a4bf5>] __do_softirq+0xf5/0x280
[<ffffffff970a4db8>] run_ksoftirqd+0x38/0x50
[<ffffffff970ce8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff970ce770>] ? lg_double_unlock+0x40/0x40
[<ffffffff970c5e61>] kthread+0xd1/0xe0
[<ffffffff970c5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffff97795df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff970c5d90>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 33 45
Complete report #323426
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc036d3be
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc038f71e
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03df2e0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb22e4860
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb22edf8c
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffb2153f39
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffb2153fb6
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffb22ec4e8
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffb22ee1a0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffb22ee2f5
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffb22ee568
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffffb1e7dfe9
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x109
|
13
|
|
vmlinux |
0xffffffffb1e7dff5
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x115
|
14
|
scsi_finish_command
|
vmlinux |
0xffffffffb22e385c
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xdc
|
15
|
scsi_softirq_done
|
vmlinux |
0xffffffffb22edab0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x130
|
16
|
blk_done_softirq
|
vmlinux |
0xffffffffb215dd36
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x96
|
17
|
__do_softirq
|
vmlinux |
0xffffffffb1ea4bf5
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xf5
|
18
|
run_ksoftirqd
|
vmlinux |
0xffffffffb1ea4db8
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x38
|
19
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb1ece8b4
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x144
|
20
|
|
vmlinux |
0xffffffffb1ece770
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x40
|
21
|
kthread
|
vmlinux |
0xffffffffb1ec5e61
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xd1
|
22
|
|
vmlinux |
0xffffffffb1ec5d90
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x40
|
23
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb2595de4
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xe
|
24
|
|
vmlinux |
0xffffffffb1ec5d90
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 592s! [ksoftirqd/3:24]
Modules linked in: tcp_lp rfcomm xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel ppdev vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus btusb joydev btrtl snd_seq btbcm btintel pcspkr snd_seq_device
bluetooth snd_pcm rfkill snd_timer snd soundcore sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel drm_kms_helper syscopyarea ata_piix sysfillrect sysimgblt serio_raw fb_sys_fops ttm e1000 drm nfit libata mptspi libnvdimm scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G L ------------ 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff98b1f8aeb180 ti: ffff98b1f8b0c000 task.ti: ffff98b1f8b0c000
RIP: 0010:[<ffffffffc036d3be>] [<ffffffffc036d3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff98b1f8b0faa0 EFLAGS: 00000246
RAX: ffffa97c80380000 RBX: ffffffffb1ec6f7b RCX: 0000000000000000
RDX: ffff98b1b541a800 RSI: ffff98b1f83b7000 RDI: 000000000000000e
RBP: ffff98b1f8b0fab0 R08: 0000000000000003 R09: 000000006e5fbfff
R10: 00000000f5964501 R11: ffff98b1f5967640 R12: ffffffffb1edae02
R13: ffff98b1f8b0fa18 R14: ffff98b1fffb1530 R15: ffff98b1f8b0fac0
FS: 0000000000000000(0000) GS:ffff98b1fdec0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007faa15ded2f0 CR3: 0000000025556000 CR4: 00000000003407e0
Call Trace:
[<ffffffffc038f71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03df2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb22e4860>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb22edf8c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb2153f39>] __blk_run_queue+0x39/0x50
[<ffffffffb2153fb6>] blk_run_queue+0x26/0x40
[<ffffffffb22ec4e8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb22ee1a0>] scsi_next_command+0x20/0x40
[<ffffffffb22ee2f5>] scsi_end_request+0x135/0x1e0
[<ffffffffb22ee568>] scsi_io_completion+0x168/0x720
[<ffffffffb1e7dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffffb1e7dff5>] ? switch_mm_irqs_off+0x115/0x290
[<ffffffffb22e385c>] scsi_finish_command+0xdc/0x140
[<ffffffffb22edab0>] scsi_softirq_done+0x130/0x160
[<ffffffffb215dd36>] blk_done_softirq+0x96/0xc0
[<ffffffffb1ea4bf5>] __do_softirq+0xf5/0x280
[<ffffffffb1ea4db8>] run_ksoftirqd+0x38/0x50
[<ffffffffb1ece8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb1ece770>] ? lg_double_unlock+0x40/0x40
[<ffffffffb1ec5e61>] kthread+0xd1/0xe0
[<ffffffffb1ec5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffffb2595de4>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffffb1ec5d90>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 37
Complete report #322757
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc036d3be
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc038f71e
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03df2e0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb22e4860
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb22edf8c
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffb2153f39
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffb2153fb6
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffb22ec4e8
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffb22ee1a0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffb22ee2f5
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffb22ee568
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffffb1e7dfe9
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x109
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffffb22e385c
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffffb22edab0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x130
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffffb215dd36
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffffb1ea4bf5
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffffb1ea4db8
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb1ece8b4
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffffb1ece770
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffffb1ec5e61
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffffb1ec5d90
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb2595de4
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xe
|
23
|
|
vmlinux |
0xffffffffb1ec5d90
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 364s! [ksoftirqd/3:24]
Modules linked in: tcp_lp rfcomm xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel ppdev vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus btusb joydev btrtl snd_seq btbcm btintel pcspkr snd_seq_device
bluetooth snd_pcm rfkill snd_timer snd soundcore sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel drm_kms_helper syscopyarea ata_piix sysfillrect sysimgblt serio_raw fb_sys_fops ttm e1000 drm nfit libata mptspi libnvdimm scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G L ------------ 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff98b1f8aeb180 ti: ffff98b1f8b0c000 task.ti: ffff98b1f8b0c000
RIP: 0010:[<ffffffffc036d3be>] [<ffffffffc036d3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff98b1f8b0faa0 EFLAGS: 00000246
RAX: ffffa97c80380000 RBX: ffff98b181eee740 RCX: 0000000000000017
RDX: ffff98b1b541a800 RSI: ffff98b1f83b7000 RDI: 000000000000000e
RBP: ffff98b1f8b0fab0 R08: 0000000000000003 R09: 000000000bdbdfff
R10: ffff98b1fdee3a30 R11: ffffd09d4019ad00 R12: 000000010000001b
R13: ffff98b18639e400 R14: ffff98b1f8b0fa14 R15: ffffffffb215c8fb
FS: 0000000000000000(0000) GS:ffff98b1fdec0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f144d9ff2e0 CR3: 0000000035ff2000 CR4: 00000000003407e0
Call Trace:
[<ffffffffc038f71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03df2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb22e4860>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb22edf8c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb2153f39>] __blk_run_queue+0x39/0x50
[<ffffffffb2153fb6>] blk_run_queue+0x26/0x40
[<ffffffffb22ec4e8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb22ee1a0>] scsi_next_command+0x20/0x40
[<ffffffffb22ee2f5>] scsi_end_request+0x135/0x1e0
[<ffffffffb22ee568>] scsi_io_completion+0x168/0x720
[<ffffffffb1e7dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffffb22e385c>] scsi_finish_command+0xdc/0x140
[<ffffffffb22edab0>] scsi_softirq_done+0x130/0x160
[<ffffffffb215dd36>] blk_done_softirq+0x96/0xc0
[<ffffffffb1ea4bf5>] __do_softirq+0xf5/0x280
[<ffffffffb1ea4db8>] run_ksoftirqd+0x38/0x50
[<ffffffffb1ece8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb1ece770>] ? lg_double_unlock+0x40/0x40
[<ffffffffb1ec5e61>] kthread+0xd1/0xe0
[<ffffffffb1ec5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffffb2595de4>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffffb1ec5d90>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 37
Complete report #322609
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff9f51b445
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x15
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc0083622
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x4a2
|
3
|
|
vmlinux |
0xffffffff9f2a8630
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x170
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff9f2a0140
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff9f2a96ac
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x4cc
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff9f11d5e9
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff9f11d666
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff9f2a7c48
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff9f2a98a0
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff9f2a99f5
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff9f2a9c68
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x168
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffff9f29f14c
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xdc
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffff9f2a91b2
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x132
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffff9f1272a6
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x96
|
15
|
__do_softirq
|
vmlinux |
0xffffffff9ee9dba5
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xf5
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffff9ee9dd68
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffff9eec6774
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x144
|
18
|
|
vmlinux |
0xffffffff9eec6630
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffff9eebdf21
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xd1
|
20
|
|
vmlinux |
0xffffffff9eebde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
21
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff9f5255f7
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x21
|
22
|
|
vmlinux |
0xffffffff9eebde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 26s! [ksoftirqd/0:3]
Modules linked in: talpa_vfshook(OE) talpa_pedconnector(OE) talpa_pedevice(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) nf_nat_ftp xt_REDIRECT nf_nat_redirect nf_conntrack_ftp ip6table_nat nf_nat_ipv6 iptable_raw xt_connlimit iptable_mangle iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat binfmt_misc tcp_diag inet_diag ip6t_REJECT nf_reject_ipv6 nf_log_ipv6 ip6table_filter nf_log_ipv4 nf_log_common xt_conntrack xt_LOG xt_limit xt_multiport ip6table_mangle ip6table_raw nf_conntrack_ipv6 nf_defrag_ipv6 nf_conntrack ip6_tables ipt_REJECT nf_reject_ipv4 iptable_filter sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr joydev vmw_balloon sg parport_pc parport vmw_vmci
shpchp i2c_piix4 tcp_htcp ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm crct10dif_common crc32c_intel e1000 drm serio_raw ata_piix libata vmw_pvscsi i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: talpa_syscallhook]
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G W OEL ------------ 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff949afa2b9fa0 ti: ffff949afa334000 task.ti: ffff949afa334000
RIP: 0010:[<ffffffff9f51b445>] [<ffffffff9f51b445>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0000:ffff949afa337b08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff949afa337ac8 RCX: ffffffff9fa30880
RDX: ffffaa0dc1b80000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff949afa337b08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff9499b66488d8 R11: ffff949d4638e800 R12: 0000000000000020
R13: ffff949ac5579800 R14: ffffffff9f2a7389 R15: ffff949afa337a80
FS: 0000000000000000(0000) GS:ffff949dbfc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff36a9f6000 CR3: 00000003c640e000 CR4: 00000000000407f0
Call Trace:
[<ffffffffc0083622>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff9f2a8630>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff9f2a0140>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9f2a96ac>] scsi_request_fn+0x4cc/0x680
[<ffffffff9f11d5e9>] __blk_run_queue+0x39/0x50
[<ffffffff9f11d666>] blk_run_queue+0x26/0x40
[<ffffffff9f2a7c48>] scsi_run_queue+0x258/0x2f0
[<ffffffff9f2a98a0>] scsi_next_command+0x20/0x40
[<ffffffff9f2a99f5>] scsi_end_request+0x135/0x1e0
[<ffffffff9f2a9c68>] scsi_io_completion+0x168/0x6a0
[<ffffffff9f29f14c>] scsi_finish_command+0xdc/0x140
[<ffffffff9f2a91b2>] scsi_softirq_done+0x132/0x160
[<ffffffff9f1272a6>] blk_done_softirq+0x96/0xc0
[<ffffffff9ee9dba5>] __do_softirq+0xf5/0x280
[<ffffffff9ee9dd68>] run_ksoftirqd+0x38/0x50
[<ffffffff9eec6774>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9eec6630>] ? lg_double_unlock+0x40/0x40
[<ffffffff9eebdf21>] kthread+0xd1/0xe0
[<ffffffff9eebde50>] ? insert_kthread_work+0x40/0x40
[<ffffffff9f5255f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9eebde50>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #257877
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc03283be
Build id: 3.10.0-1160.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc039e71e
Build id: 3.10.0-1160.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc04072e0
Build id: 3.10.0-1160.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8aee3980
Build id: 3.10.0-1160.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8aeed0ac
Build id: 3.10.0-1160.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff8ad534d9
Build id: 3.10.0-1160.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff8ad53556
Build id: 3.10.0-1160.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff8aeeb608
Build id: 3.10.0-1160.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff8aeed2c0
Build id: 3.10.0-1160.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff8aeed415
Build id: 3.10.0-1160.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff8aeed688
Build id: 3.10.0-1160.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffff8aae5f1b
Build id: 3.10.0-1160.el7.x86_64
|
0x8b
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff8aee297c
Build id: 3.10.0-1160.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff8aeecbd0
Build id: 3.10.0-1160.el7.x86_64
|
0x130
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff8ad5d226
Build id: 3.10.0-1160.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffff8aaa4b95
Build id: 3.10.0-1160.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff8aaa4d58
Build id: 3.10.0-1160.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff8aace674
Build id: 3.10.0-1160.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffff8aace530
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffff8aac5c21
Build id: 3.10.0-1160.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffff8aac5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8b193df7
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffff8aac5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 24s! [ksoftirqd/3:24]
Modules linked in: veth nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul snd_ens1371 ghash_clmulni_intel snd_rawmidi ppdev snd_ac97_codec ac97_bus snd_seq vmw_balloon aesni_intel lrw snd_seq_device
gf128mul glue_helper ablk_helper cryptd snd_pcm joydev pcspkr snd_timer snd sg i2c_piix4 soundcore vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit crct10dif_pclmul mptspi crct10dif_common crc32c_intel scsi_transport_spi ata_piix drm libnvdimm serio_raw e1000 mptscsih libata mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G W L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff8b28397d3180 ti: ffff8b28397e4000 task.ti: ffff8b28397e4000
RIP: 0010:[<ffffffffc03283be>] [<ffffffffc03283be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8b28397e7aa0 EFLAGS: 00000246
RAX: ffffaeb040440000 RBX: ffff8b27c73e6440 RCX: 0000000000000001
RDX: ffff8b27f4c02800 RSI: ffff8b2836fff000 RDI: 000000000000000e
RBP: ffff8b28397e7ab0 R08: 0000000000000003 R09: 000000006e7f3fff
R10: 00000000c60bc701 R11: ffff8b27c60bc000 R12: 0000000100000023
R13: ffff8b27c770d200 R14: ffff8b28397e7a14 R15: ffffffff8ad5bdfb
FS: 0000000000000000(0000) GS:ffff8b283b6c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f987832f060 CR3: 000000003eb8a000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc039e71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04072e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8aee3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8aeed0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffff8ad534d9>] __blk_run_queue+0x39/0x50
[<ffffffff8ad53556>] blk_run_queue+0x26/0x40
[<ffffffff8aeeb608>] scsi_run_queue+0x258/0x2f0
[<ffffffff8aeed2c0>] scsi_next_command+0x20/0x40
[<ffffffff8aeed415>] scsi_end_request+0x135/0x1e0
[<ffffffff8aeed688>] scsi_io_completion+0x168/0x720
[<ffffffff8aae5f1b>] ? put_prev_entity+0x8b/0x400
[<ffffffff8aee297c>] scsi_finish_command+0xdc/0x140
[<ffffffff8aeecbd0>] scsi_softirq_done+0x130/0x160
[<ffffffff8ad5d226>] blk_done_softirq+0x96/0xc0
[<ffffffff8aaa4b95>] __do_softirq+0xf5/0x280
[<ffffffff8aaa4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff8aace674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8aace530>] ? lg_double_unlock+0x40/0x40
[<ffffffff8aac5c21>] kthread+0xd1/0xe0
[<ffffffff8aac5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8b193df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8aac5b50>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 63 33
Complete report #241671
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc00503be
Build id: 3.10.0-693.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc009e729
Build id: 3.10.0-693.el7.x86_64
|
0x249
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc006b2b0
Build id: 3.10.0-693.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8146540a
Build id: 3.10.0-693.el7.x86_64
|
0xaa
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8146e1cf
Build id: 3.10.0-693.el7.x86_64
|
0x4df
|
6
|
|
vmlinux |
0xffffffff810ba588
Build id: 3.10.0-693.el7.x86_64
|
0x58
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff812f71d3
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
8
|
blk_run_queue
|
vmlinux |
0xffffffff812f7246
Build id: 3.10.0-693.el7.x86_64
|
0x26
|
9
|
scsi_run_queue
|
vmlinux |
0xffffffff8146c6e8
Build id: 3.10.0-693.el7.x86_64
|
0x258
|
10
|
scsi_next_command
|
vmlinux |
0xffffffff8146e3b0
Build id: 3.10.0-693.el7.x86_64
|
0x20
|
11
|
scsi_end_request
|
vmlinux |
0xffffffff8146e505
Build id: 3.10.0-693.el7.x86_64
|
0x135
|
12
|
scsi_io_completion
|
vmlinux |
0xffffffff8146e778
Build id: 3.10.0-693.el7.x86_64
|
0x168
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff81464425
Build id: 3.10.0-693.el7.x86_64
|
0xd5
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff8146dcc2
Build id: 3.10.0-693.el7.x86_64
|
0x132
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff81300510
Build id: 3.10.0-693.el7.x86_64
|
0x90
|
16
|
__do_softirq
|
vmlinux |
0xffffffff81090b3f
Build id: 3.10.0-693.el7.x86_64
|
0xef
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff81090d08
Build id: 3.10.0-693.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810b909f
Build id: 3.10.0-693.el7.x86_64
|
0x12f
|
19
|
|
vmlinux |
0xffffffff810b8f70
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
|
0xcf
|
21
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
22
|
ret_from_fork
|
vmlinux |
0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
|
0x58
|
23
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [ksoftirqd/7:43]
Modules linked in: tcp_lp nls_utf8 isofs fuse tun ip_set nfnetlink bridge stp llc vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd iosf_mbi soundcore crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev vmw_vmci sg i2c_piix4 nfit libnvdimm parport_pc parport shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata crct10dif_pclmul crct10dif_common crc32c_intel mptspi scsi_transport_spi mptscsih serio_raw e1000 mptbase i2c_core dm_mirror dm_region_hash
dm_log dm_mod [last unloaded: ip_tables]
CPU: 7 PID: 43 Comm: ksoftirqd/7 Tainted: G W L ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff88017cbd8fd0 ti: ffff88017cbe8000 task.ti: ffff88017cbe8000
RIP: 0010:[<ffffffffc00503be>] [<ffffffffc00503be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff88017cbebaa0 EFLAGS: 00000246
RAX: ffffc90000d00000 RBX: ffff8801e063d700 RCX: 0000000000000009
RDX: ffff88023736ea00 RSI: ffff8802310c8000 RDI: 000000000000000e
RBP: ffff88017cbebab0 R08: 0000000000000003 R09: ffff8802372af0d8
R10: ffff88022f3001c0 R11: ffffea0008bfb400 R12: 0000000100000001
R13: ffff88022822a100 R14: ffff88017cbeba14 R15: ffffffff812ff13b
FS: 0000000000000000(0000) GS:ffff8802357c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff6538e8078 CR3: 00000000019f2000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff8802310c8d30 000000000000003c ffff88017cbebb58 ffffffffc009e729
ffff8802310c8008 0000000000000092 0000000000000009 ffff88022f3001c0
0000000000000024 ffff8802310c8188 0000006000000009 ffff88022ff02b90
Call Trace:
[<ffffffffc009e729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffc006b2b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff8146540a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8146e1cf>] scsi_request_fn+0x4df/0x680
[<ffffffff810ba588>] ? __wake_up_common+0x58/0x90
[<ffffffff812f71d3>] __blk_run_queue+0x33/0x40
[<ffffffff812f7246>] blk_run_queue+0x26/0x40
[<ffffffff8146c6e8>] scsi_run_queue+0x258/0x2f0
[<ffffffff8146e3b0>] scsi_next_command+0x20/0x40
[<ffffffff8146e505>] scsi_end_request+0x135/0x1e0
[<ffffffff8146e778>] scsi_io_completion+0x168/0x6a0
[<ffffffff81464425>] scsi_finish_command+0xd5/0x130
[<ffffffff8146dcc2>] scsi_softirq_done+0x132/0x160
[<ffffffff81300510>] blk_done_softirq+0x90/0xc0
[<ffffffff81090b3f>] __do_softirq+0xef/0x280
[<ffffffff81090d08>] run_ksoftirqd+0x38/0x50
[<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 e3 05
Complete report #237120
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc02123be
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc023471e
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01562e0
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffa66decd0
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffa66e833c
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x4cc
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffa654f289
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffa654f306
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffa66e68e8
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffa66e8530
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffa66e8685
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffa66e88f8
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffffa65ff760
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xf0
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffffa66ddcdc
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffffa66e7e42
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x132
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffffa6558f96
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffffa62a5435
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffffa62a55f8
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa62ced54
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffffa62cec10
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffffa62c6321
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffffa62c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa698dd37
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffffa62c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 21s! [ksoftirqd/5:34]
Modules linked in: ip_vs_sh ip_vs_pe_sip nf_conntrack_sip ip_vs_wlc ip_vs_rr ip_vs veth nfnetlink_queue nfnetlink_log bluetooth rfkill nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) vfio_pci vfio_iommu_type1 vfio irqbypass xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd sg pcspkr joydev parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic
pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi scsi_transport_spi mptscsih mptbase drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix libata serio_raw e1000 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 5 PID: 34 Comm: ksoftirqd/5 Kdump: loaded Tainted: G W L ------------ T 3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff89ee7adc9070 ti: ffff89ee7add4000 task.ti: ffff89ee7add4000
RIP: 0010:[<ffffffffc02123be>] [<ffffffffc02123be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff89ee7add7aa0 EFLAGS: 00000246
RAX: ffffad9f419c0000 RBX: ffff89ef1b89e620 RCX: 0000000000000035
RDX: ffff89ed367c7200 RSI: ffff89ed367f1000 RDI: 000000000000000e
RBP: ffff89ee7add7ab0 R08: 0000000000000003 R09: 00000002d6976fff
R10: ffff89ed367f30d8 R11: ffff89ef6da2a840 R12: 0000000100000002
R13: ffff89ef2fce6880 R14: ffff89ee7add7a14 R15: ffffffffa6557aeb
FS: 0000000000000000(0000) GS:ffff89f032f40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000416960 CR3: 000000004ea10000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc023471e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01562e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa66decd0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa66e833c>] scsi_request_fn+0x4cc/0x680
[<ffffffffa654f289>] __blk_run_queue+0x39/0x50
[<ffffffffa654f306>] blk_run_queue+0x26/0x40
[<ffffffffa66e68e8>] scsi_run_queue+0x258/0x2f0
[<ffffffffa66e8530>] scsi_next_command+0x20/0x40
[<ffffffffa66e8685>] scsi_end_request+0x135/0x1e0
[<ffffffffa66e88f8>] scsi_io_completion+0x168/0x6a0
[<ffffffffa65ff760>] ? fbcon_add_cursor_timer+0xf0/0xf0
[<ffffffffa66ddcdc>] scsi_finish_command+0xdc/0x140
[<ffffffffa66e7e42>] scsi_softirq_done+0x132/0x160
[<ffffffffa6558f96>] blk_done_softirq+0x96/0xc0
[<ffffffffa62a5435>] __do_softirq+0xf5/0x280
[<ffffffffa62a55f8>] run_ksoftirqd+0x38/0x50
[<ffffffffa62ced54>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa62cec10>] ? lg_double_unlock+0x40/0x40
[<ffffffffa62c6321>] kthread+0xd1/0xe0
[<ffffffffa62c6250>] ? insert_kthread_work+0x40/0x40
[<ffffffffa698dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa62c6250>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 03 22
Complete report #229966
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc04873be
Build id: 3.10.0-862.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc04d671e
Build id: 3.10.0-862.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03e72e0
Build id: 3.10.0-862.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff9fe9b050
Build id: 3.10.0-862.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff9fea45bc
Build id: 3.10.0-862.el7.x86_64
|
0x4cc
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff9fd18ad9
Build id: 3.10.0-862.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff9fd18b56
Build id: 3.10.0-862.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff9fea2b58
Build id: 3.10.0-862.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff9fea47b0
Build id: 3.10.0-862.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff9fea4905
Build id: 3.10.0-862.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff9fea4b78
Build id: 3.10.0-862.el7.x86_64
|
0x168
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffff9fe9a05c
Build id: 3.10.0-862.el7.x86_64
|
0xdc
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffff9fea40c2
Build id: 3.10.0-862.el7.x86_64
|
0x132
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffff9fd22796
Build id: 3.10.0-862.el7.x86_64
|
0x96
|
15
|
__do_softirq
|
vmlinux |
0xffffffff9fa9a945
Build id: 3.10.0-862.el7.x86_64
|
0xf5
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffff9fa9ab08
Build id: 3.10.0-862.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffff9fac35e4
Build id: 3.10.0-862.el7.x86_64
|
0x144
|
18
|
|
vmlinux |
0xffffffff9fac34a0
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffff9fabae31
Build id: 3.10.0-862.el7.x86_64
|
0xd1
|
20
|
|
vmlinux |
0xffffffff9fabad60
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
21
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa011f637
Build id: 3.10.0-862.el7.x86_64
|
0x21
|
22
|
|
vmlinux |
0xffffffff9fabad60
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 28s! [ksoftirqd/2:19]
Modules linked in: esn_cfs(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm coretemp snd_timer snd soundcore iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_balloon parport_pc parport shpchp vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul
syscopyarea crct10dif_common sysfillrect sysimgblt fb_sys_fops crc32c_intel ttm ata_piix drm mptspi serio_raw scsi_transport_spi mptscsih e1000 mptbase libata i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: G W OEL ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2012
task: ffff8870bb068fd0 ti: ffff8870bb070000 task.ti: ffff8870bb070000
RIP: 0010:[<ffffffffc04873be>] [<ffffffffc04873be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8870bb073aa0 EFLAGS: 00000246
RAX: ffffb42600380000 RBX: ffff887064c7ee20 RCX: 0000000000000009
RDX: ffff887075b7a200 RSI: ffff887075b88000 RDI: 000000000000000e
RBP: ffff8870bb073ab0 R08: 0000000000000002 R09: 0000000021070fff
R10: ffff887075df10d8 R11: ffffdcbb00d6df00 R12: 0000000100000002
R13: ffff887064f8f780 R14: ffff8870bb073a14 R15: ffffffff9fd212cb
FS: 0000000000000000(0000) GS:ffff8870bc680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0535b39000 CR3: 0000000078fe4000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc04d671e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03e72e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9fe9b050>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9fea45bc>] scsi_request_fn+0x4cc/0x680
[<ffffffff9fd18ad9>] __blk_run_queue+0x39/0x50
[<ffffffff9fd18b56>] blk_run_queue+0x26/0x40
[<ffffffff9fea2b58>] scsi_run_queue+0x258/0x2f0
[<ffffffff9fea47b0>] scsi_next_command+0x20/0x40
[<ffffffff9fea4905>] scsi_end_request+0x135/0x1e0
[<ffffffff9fea4b78>] scsi_io_completion+0x168/0x6a0
[<ffffffff9fe9a05c>] scsi_finish_command+0xdc/0x140
[<ffffffff9fea40c2>] scsi_softirq_done+0x132/0x160
[<ffffffff9fd22796>] blk_done_softirq+0x96/0xc0
[<ffffffff9fa9a945>] __do_softirq+0xf5/0x280
[<ffffffff9fa9ab08>] run_ksoftirqd+0x38/0x50
[<ffffffff9fac35e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9fac34a0>] ? lg_double_unlock+0x40/0x40
[<ffffffff9fabae31>] kthread+0xd1/0xe0
[<ffffffff9fabad60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa011f637>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9fabad60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 49
Complete report #210030
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01e23be
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc023171e
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc023c2e0
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffa36e3e70
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffa36ed59c
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffa3553869
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffa35538e6
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffa36ebaf8
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffa36ed7b0
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffa36ed905
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffa36edb78
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffffa3987af2
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x5a2
|
13
|
|
vmlinux |
0xffffffffa3987ae6
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x596
|
14
|
scsi_finish_command
|
vmlinux |
0xffffffffa36e2e6c
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xdc
|
15
|
scsi_softirq_done
|
vmlinux |
0xffffffffa36ed0c0
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x130
|
16
|
blk_done_softirq
|
vmlinux |
0xffffffffa355d6c6
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x96
|
17
|
__do_softirq
|
vmlinux |
0xffffffffa32a4b35
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xf5
|
18
|
run_ksoftirqd
|
vmlinux |
0xffffffffa32a4cf8
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x38
|
19
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa32ce864
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x144
|
20
|
|
vmlinux |
0xffffffffa32ce720
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x40
|
21
|
kthread
|
vmlinux |
0xffffffffa32c5e11
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xd1
|
22
|
|
vmlinux |
0xffffffffa32c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x40
|
23
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa3994df7
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x21
|
24
|
|
vmlinux |
0xffffffffa32c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ksoftirqd/5:34]
Modules linked in: tcp_diag inet_diag rfcomm ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack talpa_vfshook(OE) talpa_pedconnector(OE) talpa_pedevice(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq iosf_mbi snd_seq_device crc32_pclmul snd_pcm ghash_clmulni_intel
btusb btrtl btbcm btintel snd_timer bluetooth snd aesni_intel ppdev lrw gf128mul glue_helper soundcore ablk_helper rfkill sg cryptd joydev pcspkr vmw_balloon parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit crct10dif_pclmul drm crct10dif_common ata_piix crc32c_intel libnvdimm libata mptspi scsi_transport_spi serio_raw mptscsih e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 5 PID: 34 Comm: ksoftirqd/5 Kdump: loaded Tainted: G OEL ------------ 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff98e12fb9e300 ti: ffff98e12fbc0000 task.ti: ffff98e12fbc0000
RIP: 0010:[<ffffffffc01e23be>] [<ffffffffc01e23be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff98e12fbc3aa0 EFLAGS: 00000246
RAX: ffffb9a741940000 RBX: ffff98e14d1ef5a0 RCX: 0000000000000017
RDX: ffff98e28ec35000 RSI: ffff98e29020b000 RDI: 000000000000000e
RBP: ffff98e12fbc3ab0 R08: 0000000000000003 R09: 00000002152a7fff
R10: 000000008e063401 R11: ffff98e28e063640 R12: 0000000100000006
R13: ffff98e146721500 R14: ffff98e12fbc3a14 R15: ffffffffa355c28b
FS: 0000000000000000(0000) GS:ffff98e292f40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2725271000 CR3: 000000014c810000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc023171e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc023c2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa36e3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa36ed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffffa3553869>] __blk_run_queue+0x39/0x50
[<ffffffffa35538e6>] blk_run_queue+0x26/0x40
[<ffffffffa36ebaf8>] scsi_run_queue+0x258/0x2f0
[<ffffffffa36ed7b0>] scsi_next_command+0x20/0x40
[<ffffffffa36ed905>] scsi_end_request+0x135/0x1e0
[<ffffffffa36edb78>] scsi_io_completion+0x168/0x720
[<ffffffffa3987af2>] ? __schedule+0x5a2/0x860
[<ffffffffa3987ae6>] ? __schedule+0x596/0x860
[<ffffffffa36e2e6c>] scsi_finish_command+0xdc/0x140
[<ffffffffa36ed0c0>] scsi_softirq_done+0x130/0x160
[<ffffffffa355d6c6>] blk_done_softirq+0x96/0xc0
[<ffffffffa32a4b35>] __do_softirq+0xf5/0x280
[<ffffffffa32a4cf8>] run_ksoftirqd+0x38/0x50
[<ffffffffa32ce864>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa32ce720>] ? lg_double_unlock+0x40/0x40
[<ffffffffa32c5e11>] kthread+0xd1/0xe0
[<ffffffffa32c5d40>] ? insert_kthread_work+0x40/0x40
[<ffffffffa3994df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa32c5d40>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 03 1f
Complete report #170253
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffa007b3be
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffa00a8729
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x249
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffa00962b0
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8141845a
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xaa
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff814213e1
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x501
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff812c7c63
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x33
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff812c7cd6
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff8141f808
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff81421690
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff81421805
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x155
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff814219e3
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x103
|
12
|
|
vmlinux |
0xffffffff8134fcd0
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xf0
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff81416d25
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xd5
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff81420eba
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x12a
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff812d1f80
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x90
|
16
|
__do_softirq
|
vmlinux |
0xffffffff81084b4f
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xef
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff81084d10
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x30
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810ae58f
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xff
|
19
|
|
vmlinux |
0xffffffff8163bb59
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x29
|
20
|
|
vmlinux |
0xffffffff810ae490
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x90
|
21
|
kthread
|
vmlinux |
0xffffffff810a5b2f
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xcf
|
22
|
|
vmlinux |
0xffffffff810a5a60
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x140
|
23
|
ret_from_fork
|
vmlinux |
0xffffffff81646a98
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x58
|
24
|
|
vmlinux |
0xffffffff810a5a60
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x140
|
BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:3]
Modules linked in: binfmt_misc vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr sg vmw_vmci i2c_piix4 shpchp parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common drm_kms_helper crc32c_intel serio_raw ttm drm ata_piix mptspi scsi_transport_spi mptscsih i2c_core mptbase libata vmxnet3 floppy
CPU: 0 PID: 3 Comm: ksoftirqd/0 Not tainted 3.10.0-327.28.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/17/2015
task: ffff880234161700 ti: ffff880234188000 task.ti: ffff880234188000
RIP: 0010:[<ffffffffa007b3be>] [<ffffffffa007b3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88023418ba80 EFLAGS: 00000246
RAX: ffffc90008f40000 RBX: ffff88023418ba50 RCX: 000000000000000c
RDX: ffff880036b15a00 RSI: ffff880233e32000 RDI: 000000000000000e
RBP: ffff88023418ba90 R08: 0000000000000003 R09: ffff8802300050d8
R10: ffff88004c382000 R11: ffffea0007f39d80 R12: 0000000100000002
R13: ffff8800afc87980 R14: ffff88023418b9f4 R15: ffffffff812d0f9b
FS: 0000000000000000(0000) GS:ffff88023fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fe07b435000 CR3: 0000000062495000 CR4: 00000000000407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880233e32d30 0000000000000048 ffff88023418bb38 ffffffffa00a8729
ffff880233e32008 040000003418baf8 000000000000000c ffff88004c382000
0000000000000030 ffff880233e32188 000000600000000c ffff880230142cb0
Call Trace:
[<ffffffffa00a8729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa00962b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff8141845a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff814213e1>] scsi_request_fn+0x501/0x770
[<ffffffff812c7c63>] __blk_run_queue+0x33/0x40
[<ffffffff812c7cd6>] blk_run_queue+0x26/0x40
[<ffffffff8141f808>] scsi_run_queue+0x258/0x2f0
[<ffffffff81421690>] scsi_next_command+0x20/0x40
[<ffffffff81421805>] scsi_end_request+0x155/0x1d0
[<ffffffff814219e3>] scsi_io_completion+0x103/0x600
[<ffffffff8134fcd0>] ? fbcon_add_cursor_timer+0xf0/0xf0
[<ffffffff81416d25>] scsi_finish_command+0xd5/0x130
[<ffffffff81420eba>] scsi_softirq_done+0x12a/0x150
[<ffffffff812d1f80>] blk_done_softirq+0x90/0xc0
[<ffffffff81084b4f>] __do_softirq+0xef/0x280
[<ffffffff81084d10>] run_ksoftirqd+0x30/0x50
[<ffffffff810ae58f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163bb59>] ? schedule+0x29/0x70
[<ffffffff810ae490>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5b2f>] kthread+0xcf/0xe0
[<ffffffff810a5a60>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81646a98>] ret_from_fork+0x58/0x90
[<ffffffff810a5a60>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 93 08
Complete report #161285
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc03df3be
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc04a871e
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc055f2e0
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffbbee3fa0
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffbbeed6cc
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffbbd53939
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffbbd539b6
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffbbeebc28
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffbbeed8e0
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffbbeeda35
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffbbeedca8
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffffbc188c62
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x5b2
|
13
|
|
vmlinux |
0xffffffffbc188c56
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x5a6
|
14
|
scsi_finish_command
|
vmlinux |
0xffffffffbbee2f9c
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xdc
|
15
|
scsi_softirq_done
|
vmlinux |
0xffffffffbbeed1f0
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x130
|
16
|
blk_done_softirq
|
vmlinux |
0xffffffffbbd5d796
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x96
|
17
|
__do_softirq
|
vmlinux |
0xffffffffbbaa4b35
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xf5
|
18
|
run_ksoftirqd
|
vmlinux |
0xffffffffbbaa4cf8
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x38
|
19
|
smpboot_thread_fn
|
vmlinux |
0xffffffffbbace7f4
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x144
|
20
|
|
vmlinux |
0xffffffffbbace6b0
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x40
|
21
|
kthread
|
vmlinux |
0xffffffffbbac5da1
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xd1
|
22
|
|
vmlinux |
0xffffffffbbac5cd0
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x40
|
23
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffbc195df7
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x21
|
24
|
|
vmlinux |
0xffffffffbbac5cd0
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) bnep vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel btusb btrtl btbcm aesni_intel btintel bluetooth ac97_bus lrw snd_seq gf128mul glue_helper ablk_helper cryptd snd_seq_device vmw_balloon snd_pcm pcspkr joydev snd_timer rfkill snd sg vmw_vmci i2c_piix4 soundcore parport_pc
parport sch_fq_codel ip_tables ext4 mbcache jbd2 vmwgfx sd_mod sr_mod crc_t10dif crct10dif_generic cdrom ata_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common crc32c_intel pata_acpi ttm mptspi scsi_transport_spi drm mptscsih nfit ata_piix libnvdimm libata serio_raw e1000 mptbase drm_panel_orientation_quirks fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Tainted: G L ------------ T 3.10.0-1160.25.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff899c7a3cd280 ti: ffff899c7af80000 task.ti: ffff899c7af80000
RIP: 0010:[<ffffffffc03df3be>] [<ffffffffc03df3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff899c7af83aa0 EFLAGS: 00000246
RAX: ffffb7d3c0240000 RBX: ffffffffbbad7422 RCX: 000000000000001c
RDX: ffff899c764e4000 RSI: ffff899c75848000 RDI: 000000000000000e
RBP: ffff899c7af83ab0 R08: 0000000000000003 R09: 00000000108a5fff
R10: ffff899c7c623b90 R11: ffffe314805a9400 R12: 0000000000000000
R13: ffff899c7c61acc0 R14: ffff899c7c61acc0 R15: ffff899c54f86300
FS: 0000000000000000(0000) GS:ffff899c7c600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000045a970 CR3: 00000000387cc000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc04a871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc055f2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffbbee3fa0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffbbeed6cc>] scsi_request_fn+0x4ac/0x680
[<ffffffffbbd53939>] __blk_run_queue+0x39/0x50
[<ffffffffbbd539b6>] blk_run_queue+0x26/0x40
[<ffffffffbbeebc28>] scsi_run_queue+0x258/0x2f0
[<ffffffffbbeed8e0>] scsi_next_command+0x20/0x40
[<ffffffffbbeeda35>] scsi_end_request+0x135/0x1e0
[<ffffffffbbeedca8>] scsi_io_completion+0x168/0x720
[<ffffffffbc188c62>] ? __schedule+0x5b2/0x870
[<ffffffffbc188c56>] ? __schedule+0x5a6/0x870
[<ffffffffbbee2f9c>] scsi_finish_command+0xdc/0x140
[<ffffffffbbeed1f0>] scsi_softirq_done+0x130/0x160
[<ffffffffbbd5d796>] blk_done_softirq+0x96/0xc0
[<ffffffffbbaa4b35>] __do_softirq+0xf5/0x280
[<ffffffffbbaa4cf8>] run_ksoftirqd+0x38/0x50
[<ffffffffbbace7f4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffbbace6b0>] ? lg_double_unlock+0x40/0x40
[<ffffffffbbac5da1>] kthread+0xd1/0xe0
[<ffffffffbbac5cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffffbc195df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffbbac5cd0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 d3 3e
Complete report #142616
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc03253be
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc02f671e
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc02ef2e0
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffa14e39d0
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffa14ed0fc
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffa1353519
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffa1353596
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffa14eb658
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffa14ed310
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffa14ed465
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffa14ed6d8
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffffa14ed6b8
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x148
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffffa14e29cc
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffffa14ecc20
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x130
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffffa135d266
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffffa10a4b95
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffffa10a4d58
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa10ce674
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffffa10ce530
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffffa10c5c21
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffffa10c5b50
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa1794df7
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffffa10c5b50
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 21s! [ksoftirqd/6:39]
Modules linked in: sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev sg parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm e1000 ata_piix crct10dif_pclmul crct10dif_common crc32c_intel libata serio_raw mptspi drm scsi_transport_spi vmxnet3 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 39 Comm: ksoftirqd/6 Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff9bd37eb43180 ti: ffff9bd37eb50000 task.ti: ffff9bd37eb50000
RIP: 0010:[<ffffffffc03253be>] [<ffffffffc03253be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff9bd37eb53aa0 EFLAGS: 00000246
RAX: ffffb38180ec0000 RBX: ffff9bd35e7a0fe0 RCX: 0000000000000017
RDX: ffff9bd23657f400 RSI: ffff9bd435ada000 RDI: 000000000000000e
RBP: ffff9bd37eb53ab0 R08: 0000000000000003 R09: 0000000090887fff
R10: 0000000097a24f01 R11: ffff9bd397a25c00 R12: 0000000100000080
R13: ffff9bd25c6be780 R14: ffff9bd37eb53a14 R15: ffffffffa135be3b
FS: 0000000000000000(0000) GS:ffff9bd43fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000080055cf78 CR3: 000000016ea10000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc02f671e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02ef2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa14e39d0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa14ed0fc>] scsi_request_fn+0x4ac/0x680
[<ffffffffa1353519>] __blk_run_queue+0x39/0x50
[<ffffffffa1353596>] blk_run_queue+0x26/0x40
[<ffffffffa14eb658>] scsi_run_queue+0x258/0x2f0
[<ffffffffa14ed310>] scsi_next_command+0x20/0x40
[<ffffffffa14ed465>] scsi_end_request+0x135/0x1e0
[<ffffffffa14ed6d8>] scsi_io_completion+0x168/0x720
[<ffffffffa14ed6b8>] ? scsi_io_completion+0x148/0x720
[<ffffffffa14e29cc>] scsi_finish_command+0xdc/0x140
[<ffffffffa14ecc20>] scsi_softirq_done+0x130/0x160
[<ffffffffa135d266>] blk_done_softirq+0x96/0xc0
[<ffffffffa10a4b95>] __do_softirq+0xf5/0x280
[<ffffffffa10a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffa10ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa10ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffffa10c5c21>] kthread+0xd1/0xe0
[<ffffffffa10c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffa1794df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa10c5b50>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 33 33
Complete report #125560
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffbb1883c5
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x15
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc005d632
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4a2
|
3
|
|
vmlinux |
0xffffffffbaeeb0a0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x170
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffbaee2990
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffbaeec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffbad527a9
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffbad52826
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffbaeea618
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffbaeec2d0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffbaeec425
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffbaeec698
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffffbb185850
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x310
|
13
|
|
vmlinux |
0xffffffffbb185844
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x304
|
14
|
scsi_finish_command
|
vmlinux |
0xffffffffbaee198c
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xdc
|
15
|
scsi_softirq_done
|
vmlinux |
0xffffffffbaeebbe0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x130
|
16
|
blk_done_softirq
|
vmlinux |
0xffffffffbad5c4f6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x96
|
17
|
__do_softirq
|
vmlinux |
0xffffffffbaaa5695
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xf5
|
18
|
run_ksoftirqd
|
vmlinux |
0xffffffffbaaa5858
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x38
|
19
|
smpboot_thread_fn
|
vmlinux |
0xffffffffbaacf0e4
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x144
|
20
|
|
vmlinux |
0xffffffffbaacefa0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
21
|
kthread
|
vmlinux |
0xffffffffbaac6691
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xd1
|
22
|
|
vmlinux |
0xffffffffbaac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
23
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffbb192d37
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x21
|
24
|
|
vmlinux |
0xffffffffbaac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [ksoftirqd/0:6]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter ipmi_devintf ipmi_msghandler vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul glue_helper ablk_helper cryptd pcspkr joydev vmw_balloon sg parport_pc parport i2c_piix4 vmw_vmci binfmt_misc ip_tables xfs libcrc32c ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix ahci
drm libahci crct10dif_pclmul crct10dif_common e1000e nfit crc32c_intel libnvdimm libata serio_raw ptp pps_core vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G L ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8e763a315230 ti: ffff8e763a34c000 task.ti: ffff8e763a34c000
RIP: 0010:[<ffffffffbb1883c5>] [<ffffffffbb1883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8e763a34fb08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff8e763a34fac8 RCX: ffffffffbb634e40
RDX: ffffa64701b90000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8e763a34fb08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8e78e9709c00 R11: ffff8e78e9928000 R12: 0000000000000020
R13: ffff8e74f57a3c80 R14: ffffffffbaee9d59 R15: ffff8e763a34fa80
FS: 0000000000000000(0000) GS:ffff8e78ffc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000006dee20 CR3: 00000003861c0000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc005d632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffbaeeb0a0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffbaee2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffbaeec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffffbad527a9>] __blk_run_queue+0x39/0x50
[<ffffffffbad52826>] blk_run_queue+0x26/0x40
[<ffffffffbaeea618>] scsi_run_queue+0x258/0x2f0
[<ffffffffbaeec2d0>] scsi_next_command+0x20/0x40
[<ffffffffbaeec425>] scsi_end_request+0x135/0x1e0
[<ffffffffbaeec698>] scsi_io_completion+0x168/0x720
[<ffffffffbb185850>] ? __schedule+0x310/0x840
[<ffffffffbb185844>] ? __schedule+0x304/0x840
[<ffffffffbaee198c>] scsi_finish_command+0xdc/0x140
[<ffffffffbaeebbe0>] scsi_softirq_done+0x130/0x160
[<ffffffffbad5c4f6>] blk_done_softirq+0x96/0xc0
[<ffffffffbaaa5695>] __do_softirq+0xf5/0x280
[<ffffffffbaaa5858>] run_ksoftirqd+0x38/0x50
[<ffffffffbaacf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffbaacefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffbaac6691>] kthread+0xd1/0xe0
[<ffffffffbaac65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffbb192d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffbaac65c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #101072
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc03353be
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc041871e
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03762e0
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8d4deb10
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8d4e817c
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x4cc
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff8d34f0c9
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff8d34f146
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff8d4e6728
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff8d4e8370
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff8d4e84c5
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff8d4e8738
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffff8d0e652b
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x8b
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff8d4ddb1c
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff8d4e7c82
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x132
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff8d358dd6
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffff8d0a5305
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff8d0a54c8
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff8d0cec24
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffff8d0ceae0
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffff8d0c61f1
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffff8d0c6120
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8d78dd37
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffff8d0c6120
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 24s! [ksoftirqd/7:46]
Modules linked in: tcp_diag udp_diag inet_diag nf_conntrack_netlink nfnetlink nf_conntrack_ftp binfmt_misc xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 ppdev vmw_balloon sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper joydev cryptd pcspkr parport_pc sg parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic vmwgfx drm_kms_helper syscopyarea pata_acpi sysfillrect sysimgblt fb_sys_fops ttm drm
mptspi ata_piix scsi_transport_spi mptscsih crct10dif_pclmul crct10dif_common libata serio_raw vmxnet3 mptbase crc32c_intel drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 7 PID: 46 Comm: ksoftirqd/7 Kdump: loaded Tainted: G L ------------ 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff91a8ba5b8000 ti: ffff91a8ba5b4000 task.ti: ffff91a8ba5b4000
RIP: 0010:[<ffffffffc03353be>] [<ffffffffc03353be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff91a8ba5b7aa0 EFLAGS: 00000246
RAX: ffff9fa2c36c0000 RBX: ffff91a8ba5b7a40 RCX: 000000000000002f
RDX: ffff91a435cecc00 RSI: ffff91ac2d7bb000 RDI: 000000000000000e
RBP: ffff91a8ba5b7ab0 R08: 0000000000000003 R09: 000000027d948fff
R10: ffff91a8ba4ce0d8 R11: ffffc1ede0fe7100 R12: ffff91a8ba5b7a20
R13: ffffffff8d0635a3 R14: ffff91a8ba5b7a10 R15: 0000000000000006
FS: 0000000000000000(0000) GS:ffff91ac3fc40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f59509cf000 CR3: 00000003c3210000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc041871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03762e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8d4deb10>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8d4e817c>] scsi_request_fn+0x4cc/0x680
[<ffffffff8d34f0c9>] __blk_run_queue+0x39/0x50
[<ffffffff8d34f146>] blk_run_queue+0x26/0x40
[<ffffffff8d4e6728>] scsi_run_queue+0x258/0x2f0
[<ffffffff8d4e8370>] scsi_next_command+0x20/0x40
[<ffffffff8d4e84c5>] scsi_end_request+0x135/0x1e0
[<ffffffff8d4e8738>] scsi_io_completion+0x168/0x6a0
[<ffffffff8d0e652b>] ? put_prev_entity+0x8b/0x400
[<ffffffff8d4ddb1c>] scsi_finish_command+0xdc/0x140
[<ffffffff8d4e7c82>] scsi_softirq_done+0x132/0x160
[<ffffffff8d358dd6>] blk_done_softirq+0x96/0xc0
[<ffffffff8d0a5305>] __do_softirq+0xf5/0x280
[<ffffffff8d0a54c8>] run_ksoftirqd+0x38/0x50
[<ffffffff8d0cec24>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8d0ceae0>] ? lg_double_unlock+0x40/0x40
[<ffffffff8d0c61f1>] kthread+0xd1/0xe0
[<ffffffff8d0c6120>] ? insert_kthread_work+0x40/0x40
[<ffffffff8d78dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8d0c6120>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 33 34
Complete report #80280
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb71883c5
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x15
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc0250632
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4a2
|
3
|
|
vmlinux |
0xffffffffb6eeb0a0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x170
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb6ee2990
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb6eec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffb6d527a9
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffb6d52826
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffb6eea618
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffb6eec2d0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffb6eec425
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffb6eec698
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x168
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffffb6ee198c
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xdc
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffffb6eebbe0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x130
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffffb6d5c4f6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x96
|
15
|
__do_softirq
|
vmlinux |
0xffffffffb6aa5695
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xf5
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffffb6aa5858
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb6acf0e4
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x144
|
18
|
|
vmlinux |
0xffffffffb6acefa0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffffb6ac6691
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xd1
|
20
|
|
vmlinux |
0xffffffffb6ac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
21
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb7192d37
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x21
|
22
|
|
vmlinux |
0xffffffffb6ac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 24s! [ksoftirqd/3:24]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink ipmi_devintf ipmi_msghandler vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg ppdev pcspkr vmw_balloon parport_pc joydev parport vmw_vmci i2c_piix4 binfmt_misc ip_tables xfs libcrc32c ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci e1000e nfit
libnvdimm drm libahci ata_piix crct10dif_pclmul crct10dif_common libata crc32c_intel serio_raw vmw_pvscsi ptp pps_core drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G W L ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffffa0f9344d0000 ti: ffffa0f9344d8000 task.ti: ffffa0f9344d8000
RIP: 0010:[<ffffffffb71883c5>] [<ffffffffb71883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffffa0f9344dbb08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffa0f9344dbac8 RCX: ffffffffb7634e40
RDX: ffffc09b035b0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffffa0f9344dbb08 R08: 0000000000000001 R09: 0000000000000000
R10: ffffa0ffd94c6000 R11: ffffa0ffda0dd800 R12: 0000000000000020
R13: ffffa0f857b48300 R14: ffffffffb6ee9d59 R15: ffffa0f9344dba80
FS: 0000000000000000(0000) GS:ffffa0ffffcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000105b6f0 CR3: 000000081e1d4000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc0250632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffb6eeb0a0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffb6ee2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb6eec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffffb6d527a9>] __blk_run_queue+0x39/0x50
[<ffffffffb6d52826>] blk_run_queue+0x26/0x40
[<ffffffffb6eea618>] scsi_run_queue+0x258/0x2f0
[<ffffffffb6eec2d0>] scsi_next_command+0x20/0x40
[<ffffffffb6eec425>] scsi_end_request+0x135/0x1e0
[<ffffffffb6eec698>] scsi_io_completion+0x168/0x720
[<ffffffffb6ee198c>] scsi_finish_command+0xdc/0x140
[<ffffffffb6eebbe0>] scsi_softirq_done+0x130/0x160
[<ffffffffb6d5c4f6>] blk_done_softirq+0x96/0xc0
[<ffffffffb6aa5695>] __do_softirq+0xf5/0x280
[<ffffffffb6aa5858>] run_ksoftirqd+0x38/0x50
[<ffffffffb6acf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb6acefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb6ac6691>] kthread+0xd1/0xe0
[<ffffffffb6ac65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb7192d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb6ac65c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #67637
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8cb89845
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x15
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc03c7632
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x4a2
|
3
|
|
vmlinux |
0xffffffff8c8ec0e0
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x170
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8c8e39d0
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8c8ed0fc
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff8c753519
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff8c753596
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff8c8eb658
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff8c8ed310
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff8c8ed465
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff8c8ed6d8
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x168
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffff8c8e29cc
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xdc
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffff8c8ecc20
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x130
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffff8c75d266
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x96
|
15
|
__do_softirq
|
vmlinux |
0xffffffff8c4a4b95
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xf5
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffff8c4a4d58
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffff8c4ce674
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x144
|
18
|
|
vmlinux |
0xffffffff8c4ce530
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffff8c4c5c21
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xd1
|
20
|
|
vmlinux |
0xffffffff8c4c5b50
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x40
|
21
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8cb94df7
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x21
|
22
|
|
vmlinux |
0xffffffff8c4c5b50
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:6]
IPv4: martian source 10.1.134.255 from 10.1.134.84, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b f9 81 08 00 .......PV.....
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00 .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00 .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00 .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00 .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00 .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00 .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.183, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 be 34 26 08 00 .......PV.4&..
IPv4: martian source 10.1.134.255 from 10.1.134.183, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 be 34 26 08 00 .......PV.4&..
IPv4: martian source 10.1.134.255 from 10.1.134.183, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 be 34 26 08 00 .......PV.4&..
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm dns_resolver bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon pcspkr joydev sg parport_pc parport i2c_piix4 vmw_vmci xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper ata_piix syscopyarea ahci sysfillrect sysimgblt fb_sys_fops ttm libahci crct10dif_pclmul crct10dif_common crc32c_intel drm libata nfit libnvdimm serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: CyProtectDrv]
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G OEL ------------ 3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8e7cf2375280 ti: ffff8e7cf2f74000 task.ti: ffff8e7cf2f74000
RIP: 0010:[<ffffffff8cb89845>] [<ffffffff8cb89845>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8e7cf2f77b08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff8e7cf2f77ac8 RCX: ffffffff8d034f40
RDX: ffffb85e83130000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8e7cf2f77b08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8e7cd8bfb000 R11: ffff8e7cd83de000 R12: 0000000000000020
R13: ffff8e7f271ade00 R14: ffffffff8c8ead99 R15: ffff8e7cf2f77a80
FS: 0000000000000000(0000) GS:ffff8e839d600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd0e3d23000 CR3: 00000003b2d70000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc03c7632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8c8ec0e0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8c8e39d0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8c8ed0fc>] scsi_request_fn+0x4ac/0x680
[<ffffffff8c753519>] __blk_run_queue+0x39/0x50
[<ffffffff8c753596>] blk_run_queue+0x26/0x40
[<ffffffff8c8eb658>] scsi_run_queue+0x258/0x2f0
[<ffffffff8c8ed310>] scsi_next_command+0x20/0x40
[<ffffffff8c8ed465>] scsi_end_request+0x135/0x1e0
[<ffffffff8c8ed6d8>] scsi_io_completion+0x168/0x720
[<ffffffff8c8e29cc>] scsi_finish_command+0xdc/0x140
[<ffffffff8c8ecc20>] scsi_softirq_done+0x130/0x160
[<ffffffff8c75d266>] blk_done_softirq+0x96/0xc0
[<ffffffff8c4a4b95>] __do_softirq+0xf5/0x280
[<ffffffff8c4a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff8c4ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8c4ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffff8c4c5c21>] kthread+0xd1/0xe0
[<ffffffff8c4c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8cb94df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8c4c5b50>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #53716
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc013a3be
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc032571e
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc016f2e0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff9c6e2950
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff9c6ec07c
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff9c552769
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff9c5527e6
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff9c6ea5d8
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff9c6ec290
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff9c6ec3e5
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff9c6ec658
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffff9c2bcadd
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x2d
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff9c6e194c
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff9c6ebba0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x130
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff9c55c4b6
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffff9c2a5695
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff9c2a5858
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff9c2cf0e4
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffff9c2cefa0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffff9c2c6691
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffff9c2c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff9c992d37
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x21
|
23
|
|
vmlinux |
0xffffffff9c2c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ksoftirqd/7:45]
Modules linked in: xt_multiport xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nf_nat_ftp nf_conntrack_ftp ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp ppdev iosf_mbi crc32_pclmul vmw_balloon ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg pcspkr joydev i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c
sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi scsi_transport_spi ata_piix mptscsih drm crct10dif_pclmul crct10dif_common serio_raw libata crc32c_intel mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 7 PID: 45 Comm: ksoftirqd/7 Kdump: loaded Tainted: G L ------------ 3.10.0-1127.18.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013
task: ffff96c1367bc1c0 ti: ffff96c1367cc000 task.ti: ffff96c1367cc000
RIP: 0010:[<ffffffffc013a3be>] [<ffffffffc013a3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff96c1367cfaa0 EFLAGS: 00000246
RAX: ffffb210c6a80000 RBX: ffff96c5f62d5200 RCX: 0000000000000025
RDX: ffff96cbedfa3e00 RSI: ffff96cc31200000 RDI: 000000000000000e
RBP: ffff96c1367cfab0 R08: 0000000000000003 R09: 0000000b83d2afff
R10: ffff96c60f1e3fd0 R11: ffffe49259382f00 R12: 0000000100000001
R13: ffff96cb4d494a80 R14: ffff96c1367cfa14 R15: ffffffff9c55b08b
FS: 0000000000000000(0000) GS:ffff96c60f1c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fafc26d8068 CR3: 0000000c68e5c000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc032571e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc016f2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9c6e2950>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9c6ec07c>] scsi_request_fn+0x4ac/0x680
[<ffffffff9c552769>] __blk_run_queue+0x39/0x50
[<ffffffff9c5527e6>] blk_run_queue+0x26/0x40
[<ffffffff9c6ea5d8>] scsi_run_queue+0x258/0x2f0
[<ffffffff9c6ec290>] scsi_next_command+0x20/0x40
[<ffffffff9c6ec3e5>] scsi_end_request+0x135/0x1e0
[<ffffffff9c6ec658>] scsi_io_completion+0x168/0x720
[<ffffffff9c2bcadd>] ? queue_work_on+0x2d/0x50
[<ffffffff9c6e194c>] scsi_finish_command+0xdc/0x140
[<ffffffff9c6ebba0>] scsi_softirq_done+0x130/0x160
[<ffffffff9c55c4b6>] blk_done_softirq+0x96/0xc0
[<ffffffff9c2a5695>] __do_softirq+0xf5/0x280
[<ffffffff9c2a5858>] run_ksoftirqd+0x38/0x50
[<ffffffff9c2cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9c2cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffff9c2c6691>] kthread+0xd1/0xe0
[<ffffffff9c2c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff9c992d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9c2c65c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 83 14
Complete report #22894
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffa00773be
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffa004f729
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x249
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffa01b12b0
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff81451cba
Build id: 3.10.0-514.10.2.el7.x86_64
|
0xaa
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8145adc1
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x501
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff812eb253
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x33
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff812eb2c6
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff81459158
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff8145b070
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff8145b1c5
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff8145b3e6
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x116
|
12
|
|
vmlinux |
0xffffffff8156fd48
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x18
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff81450595
Build id: 3.10.0-514.10.2.el7.x86_64
|
0xd5
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff8145a892
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x132
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff812f5bb0
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x90
|
16
|
__do_softirq
|
vmlinux |
0xffffffff8108f2cf
Build id: 3.10.0-514.10.2.el7.x86_64
|
0xef
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff8108f498
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810b927f
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x12f
|
19
|
|
vmlinux |
0xffffffff810b9150
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x90
|
20
|
kthread
|
vmlinux |
0xffffffff810b06ff
Build id: 3.10.0-514.10.2.el7.x86_64
|
0xcf
|
21
|
|
vmlinux |
0xffffffff810b0630
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x140
|
22
|
ret_from_fork
|
vmlinux |
0xffffffff81696a58
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x58
|
23
|
|
vmlinux |
0xffffffff810b0630
Build id: 3.10.0-514.10.2.el7.x86_64
|
0x140
|
NMI watchdog: BUG: soft lockup - CPU#13 stuck for 25s! [ksoftirqd/13:80]
Modules linked in: binfmt_misc intel_powerclamp coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr sg shpchp i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common mptspi drm ata_piix crc32c_intel serio_raw scsi_transport_spi mptscsih libata vmxnet3 mptbase i2c_core floppy fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 13 PID: 80 Comm: ksoftirqd/13 Tainted: G L ------------ 3.10.0-514.10.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff880fbbca1f60 ti: ffff880fbbcac000 task.ti: ffff880fbbcac000
RIP: 0010:[<ffffffffa00773be>] [<ffffffffa00773be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff880fbbcafaa0 EFLAGS: 00000246
RAX: ffffc9000ca80000 RBX: ffff880fbbcafa70 RCX: 000000000000000f
RDX: ffff881435485400 RSI: ffff880f35674000 RDI: 000000000000000e
RBP: ffff880fbbcafab0 R08: 0000000000000003 R09: ffff8811b5c590d8
R10: ffff8810367379c0 R11: ffffea001b58fa00 R12: 0000000100000002
R13: ffff8801d7343600 R14: ffff880fbbcafa14 R15: ffffffff812f4c0b
FS: 0000000000000000(0000) GS:ffff8811b5d00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f25b89c1910 CR3: 00000000019ba000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880f35674d30 0000000000000048 ffff880fbbcafb58 ffffffffa004f729
ffff880f35674008 04000000bbcafb18 000000000000000f ffff8810367379c0
000000000000003c ffff880f35674188 000000600000000f ffff880a35202dd0
Call Trace:
[<ffffffffa004f729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa01b12b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff81451cba>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8145adc1>] scsi_request_fn+0x501/0x770
[<ffffffff812eb253>] __blk_run_queue+0x33/0x40
[<ffffffff812eb2c6>] blk_run_queue+0x26/0x40
[<ffffffff81459158>] scsi_run_queue+0x258/0x2f0
[<ffffffff8145b070>] scsi_next_command+0x20/0x40
[<ffffffff8145b1c5>] scsi_end_request+0x135/0x1e0
[<ffffffff8145b3e6>] scsi_io_completion+0x116/0x670
[<ffffffff8156fd48>] ? __netif_receive_skb+0x18/0x60
[<ffffffff81450595>] scsi_finish_command+0xd5/0x130
[<ffffffff8145a892>] scsi_softirq_done+0x132/0x160
[<ffffffff812f5bb0>] blk_done_softirq+0x90/0xc0
[<ffffffff8108f2cf>] __do_softirq+0xef/0x280
[<ffffffff8108f498>] run_ksoftirqd+0x38/0x50
[<ffffffff810b927f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b9150>] ? lg_double_unlock+0x90/0x90
[<ffffffff810b06ff>] kthread+0xcf/0xe0
[<ffffffff810b0630>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81696a58>] ret_from_fork+0x58/0x90
[<ffffffff810b0630>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 08
Complete report #19496
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffa01b13be
Build id: 3.10.0-514.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffa0112729
Build id: 3.10.0-514.el7.x86_64
|
0x249
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffa01d22b0
Build id: 3.10.0-514.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff814518da
Build id: 3.10.0-514.el7.x86_64
|
0xaa
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8145a9e1
Build id: 3.10.0-514.el7.x86_64
|
0x501
|
6
|
|
vmlinux |
0xffffffff810ba118
Build id: 3.10.0-514.el7.x86_64
|
0x58
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff812eb053
Build id: 3.10.0-514.el7.x86_64
|
0x33
|
8
|
blk_run_queue
|
vmlinux |
0xffffffff812eb0c6
Build id: 3.10.0-514.el7.x86_64
|
0x26
|
9
|
scsi_run_queue
|
vmlinux |
0xffffffff81458d78
Build id: 3.10.0-514.el7.x86_64
|
0x258
|
10
|
scsi_next_command
|
vmlinux |
0xffffffff8145ac90
Build id: 3.10.0-514.el7.x86_64
|
0x20
|
11
|
scsi_end_request
|
vmlinux |
0xffffffff8145ade5
Build id: 3.10.0-514.el7.x86_64
|
0x135
|
12
|
scsi_io_completion
|
vmlinux |
0xffffffff8145b006
Build id: 3.10.0-514.el7.x86_64
|
0x116
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff814501b5
Build id: 3.10.0-514.el7.x86_64
|
0xd5
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff8145a4b2
Build id: 3.10.0-514.el7.x86_64
|
0x132
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff812f59b0
Build id: 3.10.0-514.el7.x86_64
|
0x90
|
16
|
__do_softirq
|
vmlinux |
0xffffffff8108f0ff
Build id: 3.10.0-514.el7.x86_64
|
0xef
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff8108f2c8
Build id: 3.10.0-514.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810b90af
Build id: 3.10.0-514.el7.x86_64
|
0x12f
|
19
|
|
vmlinux |
0xffffffff810b8f80
Build id: 3.10.0-514.el7.x86_64
|
0x90
|
20
|
kthread
|
vmlinux |
0xffffffff810b052f
Build id: 3.10.0-514.el7.x86_64
|
0xcf
|
21
|
|
vmlinux |
0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
|
0x140
|
22
|
ret_from_fork
|
vmlinux |
0xffffffff81696518
Build id: 3.10.0-514.el7.x86_64
|
0x58
|
23
|
|
vmlinux |
0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
|
0x140
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:3]
Modules linked in: tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event intel_powerclamp coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel snd_ens1371 lrw gf128mul glue_helper ablk_helper ppdev snd_rawmidi snd_ac97_codec cryptd ac97_bus snd_seq snd_seq_device snd_pcm snd_timer
snd soundcore vmw_balloon pcspkr sg nfit libnvdimm parport_pc parport vmw_vmci shpchp i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_piix vmwgfx e1000 mptspi scsi_transport_spi mptscsih mptbase drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm i2c_core libata fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Not tainted 3.10.0-514.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff88003bec9f60 ti: ffff88003bfd8000 task.ti: ffff88003bfd8000
RIP: 0010:[<ffffffffa01b13be>] [<ffffffffa01b13be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88003bfdbaa0 EFLAGS: 00000246
RAX: ffffc900002c0000 RBX: ffff880037a8eec0 RCX: 000000000000000c
RDX: ffff880035a30600 RSI: ffff880035a38000 RDI: 000000000000000e
RBP: ffff88003bfdbab0 R08: 0000000000000003 R09: 000000001cc31fff
R10: ffff8800001e2bc0 R11: ffffea0000d65600 R12: 0000000100000037
R13: ffff88000fff8c00 R14: ffff88003bfdba14 R15: ffffffff812f4a0b
FS: 0000000000000000(0000) GS:ffff88003c600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f024b379000 CR3: 0000000033d75000 CR4: 00000000003407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880035a38d30 0000000000000030 ffff88003bfdbb58 ffffffffa0112729
ffff880035a38008 04000000783f4b4a 000000000000000c ffff8800001e2bc0
0000000000000030 ffff880035a38188 000000600000000c ffff88000000000b
Call Trace:
[<ffffffffa0112729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa01d22b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff814518da>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8145a9e1>] scsi_request_fn+0x501/0x770
[<ffffffff810ba118>] ? __wake_up_common+0x58/0x90
[<ffffffff812eb053>] __blk_run_queue+0x33/0x40
[<ffffffff812eb0c6>] blk_run_queue+0x26/0x40
[<ffffffff81458d78>] scsi_run_queue+0x258/0x2f0
[<ffffffff8145ac90>] scsi_next_command+0x20/0x40
[<ffffffff8145ade5>] scsi_end_request+0x135/0x1e0
[<ffffffff8145b006>] scsi_io_completion+0x116/0x670
[<ffffffff814501b5>] scsi_finish_command+0xd5/0x130
[<ffffffff8145a4b2>] scsi_softirq_done+0x132/0x160
[<ffffffff812f59b0>] blk_done_softirq+0x90/0xc0
[<ffffffff8108f0ff>] __do_softirq+0xef/0x280
[<ffffffff8108f2c8>] run_ksoftirqd+0x38/0x50
[<ffffffff810b90af>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f80>] ? lg_double_unlock+0x90/0x90
[<ffffffff810b052f>] kthread+0xcf/0xe0
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81696518>] ret_from_fork+0x58/0x90
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 f3 1b
Complete report #16208
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffa011d3be
Build id: 3.10.0-327.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffa008f729
Build id: 3.10.0-327.el7.x86_64
|
0x249
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffa00462b0
Build id: 3.10.0-327.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff81417b1a
Build id: 3.10.0-327.el7.x86_64
|
0xaa
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff81420aa1
Build id: 3.10.0-327.el7.x86_64
|
0x501
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff812c73e3
Build id: 3.10.0-327.el7.x86_64
|
0x33
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff812c7456
Build id: 3.10.0-327.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff8141eec8
Build id: 3.10.0-327.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff81420d50
Build id: 3.10.0-327.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff81420ec5
Build id: 3.10.0-327.el7.x86_64
|
0x155
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff814210a3
Build id: 3.10.0-327.el7.x86_64
|
0x103
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffff814163e5
Build id: 3.10.0-327.el7.x86_64
|
0xd5
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffff8142057a
Build id: 3.10.0-327.el7.x86_64
|
0x12a
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffff812d16a0
Build id: 3.10.0-327.el7.x86_64
|
0x90
|
15
|
__do_softirq
|
vmlinux |
0xffffffff81084b0f
Build id: 3.10.0-327.el7.x86_64
|
0xef
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffff81084cd0
Build id: 3.10.0-327.el7.x86_64
|
0x30
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810ae52f
Build id: 3.10.0-327.el7.x86_64
|
0xff
|
18
|
|
vmlinux |
0xffffffff8163a909
Build id: 3.10.0-327.el7.x86_64
|
0x29
|
19
|
|
vmlinux |
0xffffffff810ae430
Build id: 3.10.0-327.el7.x86_64
|
0x90
|
20
|
kthread
|
vmlinux |
0xffffffff810a5aef
Build id: 3.10.0-327.el7.x86_64
|
0xcf
|
21
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
|
0x140
|
22
|
ret_from_fork
|
vmlinux |
0xffffffff81645858
Build id: 3.10.0-327.el7.x86_64
|
0x58
|
23
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
|
0x140
|
BUG: soft lockup - CPU#1 stuck for 24s! [ksoftirqd/1:21]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_lsm_pinned_10703(E) crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg ppdev pcspkr vmw_balloon vmxnet3 parport_pc i2c_piix4 parport vmw_vmci shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel serio_raw e1000 vmwgfx mptspi drm_kms_helper scsi_transport_spi mptscsih
ttm mptbase ata_piix drm libata i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 21 Comm: ksoftirqd/1 Tainted: P E ------------ 3.10.0-327.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff880429610b80 ti: ffff88042961c000 task.ti: ffff88042961c000
RIP: 0010:[<ffffffffa011d3be>] [<ffffffffa011d3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88042961fa80 EFLAGS: 00000246
RAX: ffffc90009b00000 RBX: ffff88042961fa50 RCX: 0000000000000000
RDX: ffff880423c64200 RSI: ffff880035d81000 RDI: 000000000000000e
RBP: ffff88042961fa90 R08: 0000000000000003 R09: ffff880035d860d8
R10: ffff8801aad4d340 R11: ffffea0006ab5300 R12: 0000000100000080
R13: ffff88010f458300 R14: ffff88042961f9f4 R15: ffffffff812d06bb
FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc0b2043000 CR3: 000000018f440000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880035d81d30 0000000000000048 ffff88042961fb38 ffffffffa008f729
ffff880035d81008 0400000081631ada 0000000000000000 ffff8801aad4d340
0000000000000000 ffff880035d81188 0000006000000000 ffff880400000007
Call Trace:
[<ffffffffa008f729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa00462b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff81417b1a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff81420aa1>] scsi_request_fn+0x501/0x770
[<ffffffff812c73e3>] __blk_run_queue+0x33/0x40
[<ffffffff812c7456>] blk_run_queue+0x26/0x40
[<ffffffff8141eec8>] scsi_run_queue+0x258/0x2f0
[<ffffffff81420d50>] scsi_next_command+0x20/0x40
[<ffffffff81420ec5>] scsi_end_request+0x155/0x1d0
[<ffffffff814210a3>] scsi_io_completion+0x103/0x600
[<ffffffff814163e5>] scsi_finish_command+0xd5/0x130
[<ffffffff8142057a>] scsi_softirq_done+0x12a/0x150
[<ffffffff812d16a0>] blk_done_softirq+0x90/0xc0
[<ffffffff81084b0f>] __do_softirq+0xef/0x280
[<ffffffff81084cd0>] run_ksoftirqd+0x30/0x50
[<ffffffff810ae52f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163a909>] ? schedule+0x29/0x70
[<ffffffff810ae430>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5aef>] kthread+0xcf/0xe0
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81645858>] ret_from_fork+0x58/0x90
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 12
Complete report #12175
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc04793be
Build id: 3.10.0-957.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc049b71e
Build id: 3.10.0-957.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc04942e0
Build id: 3.10.0-957.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff908d0960
Build id: 3.10.0-957.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff908d9efc
Build id: 3.10.0-957.el7.x86_64
|
0x4cc
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff90742ff9
Build id: 3.10.0-957.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff90743076
Build id: 3.10.0-957.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff908d84a8
Build id: 3.10.0-957.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff908da0f0
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff908da245
Build id: 3.10.0-957.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff908da4b8
Build id: 3.10.0-957.el7.x86_64
|
0x168
|
12
|
|
vmlinux |
0xffffffff90a397f8
Build id: 3.10.0-957.el7.x86_64
|
0x18
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff908cf96c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff908d9a02
Build id: 3.10.0-957.el7.x86_64
|
0x132
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff9074cce6
Build id: 3.10.0-957.el7.x86_64
|
0x96
|
16
|
__do_softirq
|
vmlinux |
0xffffffff904a0f05
Build id: 3.10.0-957.el7.x86_64
|
0xf5
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff904a10c8
Build id: 3.10.0-957.el7.x86_64
|
0x38
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff904ca5c4
Build id: 3.10.0-957.el7.x86_64
|
0x144
|
19
|
|
vmlinux |
0xffffffff904ca480
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
20
|
kthread
|
vmlinux |
0xffffffff904c1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
21
|
|
vmlinux |
0xffffffff904c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff90b74c1d
Build id: 3.10.0-957.el7.x86_64
|
0x7
|
23
|
|
vmlinux |
0xffffffff904c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [ksoftirqd/6:40]
Modules linked in: binfmt_misc veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_nat_ipv4 xt_addrtype nf_nat br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper vmw_balloon ablk_helper cryptd joydev pcspkr sg nfit parport_pc vmw_vmci parport libnvdimm i2c_piix4 ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul drm crct10dif_common crc32c_intel serio_raw drm_panel_orientation_quirks mptspi vmxnet3 scsi_transport_spi
mptscsih mptbase floppy ata_piix libata dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 40 Comm: ksoftirqd/6 Kdump: loaded Tainted: G W L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9af3f4e51040 ti: ffff9af3f4e5c000 task.ti: ffff9af3f4e5c000
RIP: 0010:[<ffffffffc04793be>] [<ffffffffc04793be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9af3f4e5faa0 EFLAGS: 00000246
RAX: ffffb0f706a00000 RBX: ffff9af35921ffc0 RCX: 0000000000000018
RDX: ffff9afb5dfe7000 RSI: ffff9afb5dc7e000 RDI: 000000000000000e
RBP: ffff9af3f4e5fab0 R08: 0000000000000003 R09: 000000021d014fff
R10: ffff9afb5dcda0d8 R11: ffff9af775e7ebc0 R12: 000000010000007f
R13: ffff9aec546bbb00 R14: ffff9af3f4e5fa14 R15: ffffffff9074b83b
FS: 0000000000000000(0000) GS:ffff9afb7fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff1ba8be000 CR3: 0000000315410000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc049b71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04942e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff908d0960>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff908d9efc>] scsi_request_fn+0x4cc/0x680
[<ffffffff90742ff9>] __blk_run_queue+0x39/0x50
[<ffffffff90743076>] blk_run_queue+0x26/0x40
[<ffffffff908d84a8>] scsi_run_queue+0x258/0x2f0
[<ffffffff908da0f0>] scsi_next_command+0x20/0x40
[<ffffffff908da245>] scsi_end_request+0x135/0x1e0
[<ffffffff908da4b8>] scsi_io_completion+0x168/0x6a0
[<ffffffff90a397f8>] ? __netif_receive_skb+0x18/0x60
[<ffffffff908cf96c>] scsi_finish_command+0xdc/0x140
[<ffffffff908d9a02>] scsi_softirq_done+0x132/0x160
[<ffffffff9074cce6>] blk_done_softirq+0x96/0xc0
[<ffffffff904a0f05>] __do_softirq+0xf5/0x280
[<ffffffff904a10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff904ca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff904ca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff904c1c31>] kthread+0xd1/0xe0
[<ffffffff904c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff90b74c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff904c1b60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 73 48
Complete report #7798
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffa00b73be
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffa00d2729
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x249
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffa004d2b0
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8141845a
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xaa
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff814213e1
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x501
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff812c7c63
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x33
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff812c7cd6
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff8141f808
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff81421690
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff81421805
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x155
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff814219e3
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x103
|
12
|
|
vmlinux |
0xffffffff8130cd4d
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xd
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff81416d25
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xd5
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff81420eba
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x12a
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff812d1f80
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x90
|
16
|
__do_softirq
|
vmlinux |
0xffffffff81084b4f
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xef
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff81084d10
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x30
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810ae58f
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xff
|
19
|
|
vmlinux |
0xffffffff8163bb59
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x29
|
20
|
|
vmlinux |
0xffffffff810ae490
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x90
|
21
|
kthread
|
vmlinux |
0xffffffff810a5b2f
Build id: 3.10.0-327.28.3.el7.x86_64
|
0xcf
|
22
|
|
vmlinux |
0xffffffff810a5a60
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x140
|
23
|
ret_from_fork
|
vmlinux |
0xffffffff81646a98
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x58
|
24
|
|
vmlinux |
0xffffffff810a5a60
Build id: 3.10.0-327.28.3.el7.x86_64
|
0x140
|
BUG: soft lockup - CPU#1 stuck for 57s! [tuned:2323]
INFO: rcu_sched detected stalls on CPUs/tasks: {} (detected by 0, t=75895 jiffies, g=14357339, c=14357338, q=0)
INFO: Stall ended before state dump start
BUG: soft lockup - CPU#0 stuck for 57s! [ksoftirqd/0:3]
Modules linked in: ppdev coretemp vmw_balloon pcspkr sg parport_pc i2c_piix4 parport shpchp vmw_vmci ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi crc32c_intel serio_raw vmwgfx mptspi drm_kms_helper ttm scsi_transport_spi drm mptscsih vmxnet3 mptbase ata_piix i2c_core libata floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Tainted: G L ------------ 3.10.0-327.28.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff880139b11700 ti: ffff880139b90000 task.ti: ffff880139b90000
RIP: 0010:[<ffffffffa00b73be>] [<ffffffffa00b73be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff880139b93a80 EFLAGS: 00000246
RAX: ffffc900087c0000 RBX: ffff880139b93a50 RCX: 0000000000000000
RDX: ffff880036a50c00 RSI: ffff880135c5a000 RDI: 000000000000000e
RBP: ffff880139b93a90 R08: 0000000000000003 R09: ffff8800360150d8
R10: ffff88009bb28540 R11: ffffea00026eca00 R12: 0000000100000002
R13: ffff8800ae8c8300 R14: ffff880139b939f4 R15: ffffffff812d0f9b
FS: 0000000000000000(0000) GS:ffff88013fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007efee14e7000 CR3: 00000000566cd000 CR4: 00000000000007f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880135c5ad30 0000000000000048 ffff880139b93b38 ffffffffa00d2729
ffff880135c5a008 04000000d96c3de6 0000000000000000 ffff88009bb28540
0000000000000000 ffff880135c5a188 0000006000000000 ffff880036182830
Call Trace:
[<ffffffffa00d2729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa004d2b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff8141845a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff814213e1>] scsi_request_fn+0x501/0x770
[<ffffffff812c7c63>] __blk_run_queue+0x33/0x40
[<ffffffff812c7cd6>] blk_run_queue+0x26/0x40
[<ffffffff8141f808>] scsi_run_queue+0x258/0x2f0
[<ffffffff81421690>] scsi_next_command+0x20/0x40
[<ffffffff81421805>] scsi_end_request+0x155/0x1d0
[<ffffffff814219e3>] scsi_io_completion+0x103/0x600
[<ffffffff8130cd4d>] ? list_del+0xd/0x30
[<ffffffff81416d25>] scsi_finish_command+0xd5/0x130
[<ffffffff81420eba>] scsi_softirq_done+0x12a/0x150
[<ffffffff812d1f80>] blk_done_softirq+0x90/0xc0
[<ffffffff81084b4f>] __do_softirq+0xef/0x280
[<ffffffff81084d10>] run_ksoftirqd+0x30/0x50
[<ffffffff810ae58f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163bb59>] ? schedule+0x29/0x70
[<ffffffff810ae490>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5b2f>] kthread+0xcf/0xe0
[<ffffffff810a5a60>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81646a98>] ret_from_fork+0x58/0x90
[<ffffffff810a5a60>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 0c
Complete report #6584
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01173be
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc013271e
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc010b2e0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffa2ce2990
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffa2cec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4ac
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffa2b527a9
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffa2b52826
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffa2cea618
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffa2cec2d0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffa2cec425
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffa2cec698
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x168
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffffa2ce198c
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xdc
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffffa2cebbe0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x130
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffffa2b5c4f6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x96
|
15
|
__do_softirq
|
vmlinux |
0xffffffffa28a5695
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xf5
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffffa28a5858
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa28cf0e4
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x144
|
18
|
|
vmlinux |
0xffffffffa28cefa0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffffa28c6691
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xd1
|
20
|
|
vmlinux |
0xffffffffa28c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
21
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa2f92d37
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x21
|
22
|
|
vmlinux |
0xffffffffa28c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 43s! [ksoftirqd/1:14]
Modules linked in: vmfsrtx_dev(POE) vmfsrtx_dev_helper(POE) vmfsrtx_export(POE) vmfsrtx_bridge(POE) vmfsrtx_bridge_helper(OE) fuse tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd pcspkr joydev sg parport_pc parport vmw_vmci i2c_piix4 auth_rpcgss sunrpc
ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ttm crc32c_intel ata_piix drm serio_raw libata mptspi scsi_transport_spi mptscsih mptbase vmxnet3 nfit libnvdimm drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: vmfsrtx_dev_helper]
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: P OEL ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff932278ed20e0 ti: ffff932278ee4000 task.ti: ffff932278ee4000
RIP: 0010:[<ffffffffc01173be>] [<ffffffffc01173be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff932278ee7aa0 EFLAGS: 00000246
RAX: ffffafb2807c0000 RBX: ffff9321f9d83150 RCX: 0000000000000006
RDX: ffff932175efe800 RSI: ffff932175d53000 RDI: 000000000000000e
RBP: ffff932278ee7ab0 R08: 0000000000000002 R09: 000000011d5bbfff
R10: ffff93227ae63c00 R11: ffffec8044de9700 R12: 00000000005c2ed4
R13: 0000000078ee7a58 R14: ffffffffa28e5fca R15: ffff932278ee7a40
FS: 0000000000000000(0000) GS:ffff93227ae40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fdc51504000 CR3: 0000000135e34000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc013271e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc010b2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa2ce2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa2cec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffffa2b527a9>] __blk_run_queue+0x39/0x50
[<ffffffffa2b52826>] blk_run_queue+0x26/0x40
[<ffffffffa2cea618>] scsi_run_queue+0x258/0x2f0
[<ffffffffa2cec2d0>] scsi_next_command+0x20/0x40
[<ffffffffa2cec425>] scsi_end_request+0x135/0x1e0
[<ffffffffa2cec698>] scsi_io_completion+0x168/0x720
[<ffffffffa2ce198c>] scsi_finish_command+0xdc/0x140
[<ffffffffa2cebbe0>] scsi_softirq_done+0x130/0x160
[<ffffffffa2b5c4f6>] blk_done_softirq+0x96/0xc0
[<ffffffffa28a5695>] __do_softirq+0xf5/0x280
[<ffffffffa28a5858>] run_ksoftirqd+0x38/0x50
[<ffffffffa28cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa28cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa28c6691>] kthread+0xd1/0xe0
[<ffffffffa28c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2f92d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa28c65c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 12
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [CRTC:37:crtc-0] flip_done timed out
Complete report #1812
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc03f63be
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc045071e
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x24e
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc04112e0
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffadc9aff0
Build id: 3.10.0-862.2.3.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffadca455c
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x4cc
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffadb18a79
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffadb18af6
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffadca2af8
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffadca4750
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffadca48a5
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffadca4b18
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x168
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffffadc99ffc
Build id: 3.10.0-862.2.3.el7.x86_64
|
0xdc
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffffadca4062
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x132
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffffadb22736
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x96
|
15
|
__do_softirq
|
vmlinux |
0xffffffffad89a945
Build id: 3.10.0-862.2.3.el7.x86_64
|
0xf5
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffffad89ab08
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffffad8c35e4
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x144
|
18
|
|
vmlinux |
0xffffffffad8c34a0
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffffad8bae31
Build id: 3.10.0-862.2.3.el7.x86_64
|
0xd1
|
20
|
|
vmlinux |
0xffffffffad8bad60
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x40
|
21
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffadf1f5f7
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x21
|
22
|
|
vmlinux |
0xffffffffad8bad60
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 28s! [ksoftirqd/3:24]
Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ip_set nfnetlink ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_security iptable_raw tcp_lp fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev vmw_balloon pcspkr sg nfit libnvdimm parport_pc parport shpchp vmw_vmci i2c_piix4 ip_tables xfs libcrc32c
sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix mptspi scsi_transport_spi mptscsih libata drm crct10dif_pclmul crct10dif_common e1000 crc32c_intel mptbase serio_raw i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G W L ------------ 3.10.0-862.2.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9bc4bd7c1fa0 ti: ffff9bc4bd7d0000 task.ti: ffff9bc4bd7d0000
RIP: 0010:[<ffffffffc03f63be>] [<ffffffffc03f63be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9bc4bd7d3aa0 EFLAGS: 00000246
RAX: ffffad76c0e80000 RBX: ffff9bc4f3765800 RCX: 0000000000000047
RDX: ffff9bc4f45cfe00 RSI: ffff9bc4f5284000 RDI: 000000000000000e
RBP: ffff9bc4bd7d3ab0 R08: 0000000000000002 R09: 00000001b44345ff
R10: ffff9bc4f52850d8 R11: ffff9bc4f22c8540 R12: 0000000100000001
R13: ffff9bc4ed37ad80 R14: ffff9bc4bd7d3a14 R15: ffffffffadb2126b
FS: 0000000000000000(0000) GS:ffff9bc4ffd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fdd856e3080 CR3: 000000013d80e000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc045071e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04112e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffadc9aff0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffadca455c>] scsi_request_fn+0x4cc/0x680
[<ffffffffadb18a79>] __blk_run_queue+0x39/0x50
[<ffffffffadb18af6>] blk_run_queue+0x26/0x40
[<ffffffffadca2af8>] scsi_run_queue+0x258/0x2f0
[<ffffffffadca4750>] scsi_next_command+0x20/0x40
[<ffffffffadca48a5>] scsi_end_request+0x135/0x1e0
[<ffffffffadca4b18>] scsi_io_completion+0x168/0x6a0
[<ffffffffadc99ffc>] scsi_finish_command+0xdc/0x140
[<ffffffffadca4062>] scsi_softirq_done+0x132/0x160
[<ffffffffadb22736>] blk_done_softirq+0x96/0xc0
[<ffffffffad89a945>] __do_softirq+0xf5/0x280
[<ffffffffad89ab08>] run_ksoftirqd+0x38/0x50
[<ffffffffad8c35e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffad8c34a0>] ? lg_double_unlock+0x40/0x40
[<ffffffffad8bae31>] kthread+0xd1/0xe0
[<ffffffffad8bad60>] ? insert_kthread_work+0x40/0x40
[<ffffffffadf1f5f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffad8bad60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 43 40
Complete report #1024
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb1d83815
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x15
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc0301632
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x4a2
|
3
|
|
vmlinux |
0xffffffffb1ae72d0
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x170
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb1adecd0
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xb0
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb1ae833c
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x4cc
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffb194f289
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x39
|
7
|
blk_run_queue
|
vmlinux |
0xffffffffb194f306
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffffb1ae68e8
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffffb1ae8530
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffffb1ae8685
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffffb1ae88f8
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x168
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffffb1addcdc
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xdc
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffffb1ae7e42
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x132
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffffb1958f96
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x96
|
15
|
__do_softirq
|
vmlinux |
0xffffffffb16a5435
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xf5
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffffb16a55f8
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb16ced54
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x144
|
18
|
|
vmlinux |
0xffffffffb16cec10
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x40
|
19
|
kthread
|
vmlinux |
0xffffffffb16c6321
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xd1
|
20
|
|
vmlinux |
0xffffffffb16c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x40
|
21
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb1d8dd37
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x21
|
22
|
|
vmlinux |
0xffffffffb16c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ksoftirqd/1:14]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 sb_edac ppdev iosf_mbi nfit crc32_pclmul ghash_clmulni_intel vmw_balloon libnvdimm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg joydev pcspkr parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm libahci crct10dif_pclmul crct10dif_common crc32c_intel ata_piix serio_raw libata
vmxnet3 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G L ------------ 3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/11/2017
task: ffff8cf27c418000 ti: ffff8cf27c414000 task.ti: ffff8cf27c414000
RIP: 0010:[<ffffffffb1d83815>] [<ffffffffb1d83815>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8cf27c417b08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff8cf27c417ac8 RCX: ffffffffb2234800
RDX: ffffb1a500cc0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8cf27c417b08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8cf32fc1d8d8 R11: ffff8cf330575800 R12: 0000000000000020
R13: ffff8cf254595080 R14: ffffffffb1ae6029 R15: ffff8cf27c417a80
FS: 0000000000000000(0000) GS:ffff8cf335640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6936097000 CR3: 00000000014b2000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc0301632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffb1ae72d0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffb1adecd0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb1ae833c>] scsi_request_fn+0x4cc/0x680
[<ffffffffb194f289>] __blk_run_queue+0x39/0x50
[<ffffffffb194f306>] blk_run_queue+0x26/0x40
[<ffffffffb1ae68e8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb1ae8530>] scsi_next_command+0x20/0x40
[<ffffffffb1ae8685>] scsi_end_request+0x135/0x1e0
[<ffffffffb1ae88f8>] scsi_io_completion+0x168/0x6a0
[<ffffffffb1addcdc>] scsi_finish_command+0xdc/0x140
[<ffffffffb1ae7e42>] scsi_softirq_done+0x132/0x160
[<ffffffffb1958f96>] blk_done_softirq+0x96/0xc0
[<ffffffffb16a5435>] __do_softirq+0xf5/0x280
[<ffffffffb16a55f8>] run_ksoftirqd+0x38/0x50
[<ffffffffb16ced54>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb16cec10>] ? lg_double_unlock+0x40/0x40
[<ffffffffb16c6321>] kthread+0xd1/0xe0
[<ffffffffb16c6250>] ? insert_kthread_work+0x40/0x40
[<ffffffffb1d8dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb16c6250>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #161
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffa00773be
Build id: 3.10.0-514.el7.x86_64
|
0x5e
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffa0092729
Build id: 3.10.0-514.el7.x86_64
|
0x249
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffa01572b0
Build id: 3.10.0-514.el7.x86_64
|
0x50
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff814518da
Build id: 3.10.0-514.el7.x86_64
|
0xaa
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8145a9e1
Build id: 3.10.0-514.el7.x86_64
|
0x501
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff812eb053
Build id: 3.10.0-514.el7.x86_64
|
0x33
|
7
|
blk_run_queue
|
vmlinux |
0xffffffff812eb0c6
Build id: 3.10.0-514.el7.x86_64
|
0x26
|
8
|
scsi_run_queue
|
vmlinux |
0xffffffff81458d78
Build id: 3.10.0-514.el7.x86_64
|
0x258
|
9
|
scsi_next_command
|
vmlinux |
0xffffffff8145ac90
Build id: 3.10.0-514.el7.x86_64
|
0x20
|
10
|
scsi_end_request
|
vmlinux |
0xffffffff8145ade5
Build id: 3.10.0-514.el7.x86_64
|
0x135
|
11
|
scsi_io_completion
|
vmlinux |
0xffffffff8145b006
Build id: 3.10.0-514.el7.x86_64
|
0x116
|
12
|
scsi_finish_command
|
vmlinux |
0xffffffff814501b5
Build id: 3.10.0-514.el7.x86_64
|
0xd5
|
13
|
scsi_softirq_done
|
vmlinux |
0xffffffff8145a4b2
Build id: 3.10.0-514.el7.x86_64
|
0x132
|
14
|
blk_done_softirq
|
vmlinux |
0xffffffff812f59b0
Build id: 3.10.0-514.el7.x86_64
|
0x90
|
15
|
__do_softirq
|
vmlinux |
0xffffffff8108f0ff
Build id: 3.10.0-514.el7.x86_64
|
0xef
|
16
|
run_ksoftirqd
|
vmlinux |
0xffffffff8108f2c8
Build id: 3.10.0-514.el7.x86_64
|
0x38
|
17
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810b90af
Build id: 3.10.0-514.el7.x86_64
|
0x12f
|
18
|
|
vmlinux |
0xffffffff810b8f80
Build id: 3.10.0-514.el7.x86_64
|
0x90
|
19
|
kthread
|
vmlinux |
0xffffffff810b052f
Build id: 3.10.0-514.el7.x86_64
|
0xcf
|
20
|
|
vmlinux |
0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
|
0x140
|
21
|
ret_from_fork
|
vmlinux |
0xffffffff81696518
Build id: 3.10.0-514.el7.x86_64
|
0x58
|
22
|
|
vmlinux |
0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
|
0x140
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ksoftirqd/2:18]
Modules linked in: talpa_vfshook(OE) talpa_pedconnector(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) talpa_pedevice(OE) binfmt_misc tcp_diag inet_diag ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock intel_powerclamp coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon sg pcspkr
parport_pc shpchp parport vmw_vmci i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt ata_piix fb_sys_fops ttm libata crct10dif_pclmul mptspi crct10dif_common drm crc32c_intel serio_raw scsi_transport_spi vmxnet3 mptscsih mptbase i2c_core floppy fjes dm_mirror dm_region_hash dm_log dm_mod [last unloaded: talpa_syscallhook]
CPU: 2 PID: 18 Comm: ksoftirqd/2 Tainted: G OEL ------------ 3.10.0-514.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff880182278000 ti: ffff880182280000 task.ti: ffff880182280000
RIP: 0010:[<ffffffffa00773be>] [<ffffffffa00773be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff880182283aa0 EFLAGS: 00000246
RAX: ffffc90001c00000 RBX: ffff880182283a70 RCX: 000000000000000d
RDX: ffff88042b8bde00 RSI: ffff88042c125000 RDI: 000000000000000e
RBP: ffff880182283ab0 R08: 0000000000000003 R09: ffff88042bbe30d8
R10: ffff88042a675340 R11: ffffea0010a99d00 R12: 0000000100000080
R13: ffff88040f6e0a80 R14: ffff880182283a14 R15: ffffffff812f4a0b
FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fdd213cb000 CR3: 00000000019ba000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88042c125d30 0000000000000048 ffff880182283b58 ffffffffa0092729
ffff88042c125008 0400000082283b18 000000000000000d ffff88042a675340
0000000000000034 ffff88042c125188 000000600000000d ffff880400000007
Call Trace:
[<ffffffffa0092729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa01572b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff814518da>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8145a9e1>] scsi_request_fn+0x501/0x770
[<ffffffff812eb053>] __blk_run_queue+0x33/0x40
[<ffffffff812eb0c6>] blk_run_queue+0x26/0x40
[<ffffffff81458d78>] scsi_run_queue+0x258/0x2f0
[<ffffffff8145ac90>] scsi_next_command+0x20/0x40
[<ffffffff8145ade5>] scsi_end_request+0x135/0x1e0
[<ffffffff8145b006>] scsi_io_completion+0x116/0x670
[<ffffffff814501b5>] scsi_finish_command+0xd5/0x130
[<ffffffff8145a4b2>] scsi_softirq_done+0x132/0x160
[<ffffffff812f59b0>] blk_done_softirq+0x90/0xc0
[<ffffffff8108f0ff>] __do_softirq+0xef/0x280
[<ffffffff8108f2c8>] run_ksoftirqd+0x38/0x50
[<ffffffff810b90af>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f80>] ? lg_double_unlock+0x90/0x90
[<ffffffff810b052f>] kthread+0xcf/0xe0
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81696518>] ret_from_fork+0x58/0x90
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 08