Info

Permalink
Function
mpt_put_msg_frame
First occurrence
2020-09-22
Last occurrence
2025-04-25
State
NEW
Components
kernel
Quality
-47

Statistics

Operating system Count
CentOS 7 3,791
Architecture Count
x86_64 3,791
Related packages Count
kernel 3,791
 0:3.10.0-1062.el7 354
 0:3.10.0-862.el7 269
 0:3.10.0-514.el7 200
 0:3.10.0-514.26.2.el7 198
 0:3.10.0-1160.el7 189
 0:3.10.0-1160.6.1.el7 179
 0:3.10.0-957.1.3.el7 175
 0:3.10.0-693.2.2.el7 164
 0:3.10.0-957.el7 162
 0:3.10.0-1127.19.1.el7 144
 0:3.10.0-1062.9.1.el7 121
 0:3.10.0-1160.88.1.el7 94
 0:3.10.0-693.el7 94
 0:3.10.0-1160.42.2.el7 72
 0:3.10.0-1160.81.1.el7 66
 0:3.10.0-1160.25.1.el7 62
 0:3.10.0-1160.59.1.el7 59
 0:3.10.0-1160.53.1.el7 58
 0:3.10.0-1160.71.1.el7 55
 0:3.10.0-1160.15.2.el7 48
 0:3.10.0-1062.12.1.el7 48
 0:3.10.0-957.21.3.el7 48
 0:3.10.0-1062.18.1.el7 46
 0:3.10.0-1127.el7 46
 0:3.10.0-514.10.2.el7 45
 0:3.10.0-1160.49.1.el7 44
 0:3.10.0-1160.31.1.el7 40
 0:3.10.0-1160.11.1.el7 39
 0:3.10.0-1160.76.1.el7 38
 0:3.10.0-1127.10.1.el7 36
 0:3.10.0-1160.83.1.el7 34
 0:3.10.0-1160.21.1.el7 34
 0:3.10.0-1160.108.1.el7 33
 0:3.10.0-1062.1.2.el7 29
 0:3.10.0-1160.62.1.el7 28
 0:3.10.0-957.27.2.el7 28
 0:3.10.0-1160.45.1.el7 26
 0:3.10.0-1127.18.2.el7 26
 0:3.10.0-1127.13.1.el7 24
 0:3.10.0-327.36.3.el7 24
 0:3.10.0-957.12.2.el7 24
 0:3.10.0-327.36.2.el7 22
 0:3.10.0-1160.118.1.el7 19
 0:3.10.0-1062.1.1.el7 19
 0:3.10.0-327.36.1.el7 19
 0:3.10.0-862.2.3.el7 16
 0:3.10.0-862.11.6.el7 15
 0:3.10.0-957.5.1.el7 15
 0:3.10.0-1160.36.2.el7 14
 0:3.10.0-1160.80.1.el7 13
 0:3.10.0-862.6.3.el7 12
 0:3.10.0-1160.119.1.el7 11
 0:3.10.0-693.21.1.el7 10
 0:3.10.0-1160.66.1.el7 10
 0:3.10.0-1160.90.1.el7 9
 0:3.10.0-1127.8.2.el7 9
 0:3.10.0-1160.105.1.el7 9
 0:3.10.0-327.el7 8
 0:3.10.0-327.28.3.el7 8
 0:3.10.0-1160.95.1.el7 7
 0:3.10.0-1160.92.1.el7 5
 0:3.10.0-327.22.2.el7 5
 0:3.10.0-1160.102.1.el7 4
 0:3.10.0-1160.99.1.el7 4
 0:3.10.0-862.14.4.el7 4
 0:3.10.0-957.12.1.el7 3
 0:3.10.0-1160.41.1.el7 3
 0:3.10.0-693.11.1.el7 2
 0:3.10.0-1062.4.1.el7 2
 0:3.10.0-1160.114.2.el7 2
 0:3.10.0-862.9.1.el7 2
 0:3.10.0-957.10.1.el7 2
 0:3.10.0-1160.24.1.el7 1
 0:3.10.0-1062.4.3.el7 1
 0:3.10.0-514.16.1.el7 1
 0:3.10.0-862.3.2.el7 1
 0:3.10.0-693.17.1.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

Report backtrace


Complete report #2060
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc00a93be
Build id: 3.10.0-957.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc00c471e
Build id: 3.10.0-957.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc010c2e0
Build id: 3.10.0-957.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa1ed0960
Build id: 3.10.0-957.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffa1ed9efc
Build id: 3.10.0-957.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffffa1d42ff9
Build id: 3.10.0-957.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffa1d43076
Build id: 3.10.0-957.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffa1ed84a8
Build id: 3.10.0-957.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffa1eda0f0
Build id: 3.10.0-957.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffa1eda245
Build id: 3.10.0-957.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffa1eda4b8
Build id: 3.10.0-957.el7.x86_64
0x168
12
scsi_finish_command
vmlinux 0xffffffffa1ecf96c
Build id: 3.10.0-957.el7.x86_64
0xdc
13
scsi_softirq_done
vmlinux 0xffffffffa1ed9a02
Build id: 3.10.0-957.el7.x86_64
0x132
14
blk_done_softirq
vmlinux 0xffffffffa1d4cce6
Build id: 3.10.0-957.el7.x86_64
0x96
15
__do_softirq
vmlinux 0xffffffffa1aa0f05
Build id: 3.10.0-957.el7.x86_64
0xf5
16
run_ksoftirqd
vmlinux 0xffffffffa1aa10c8
Build id: 3.10.0-957.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffffa1aca5c4
Build id: 3.10.0-957.el7.x86_64
0x144
18
? lg_double_unlock
vmlinux 0xffffffffa1aca480
Build id: 3.10.0-957.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffffa1ac1c31
Build id: 3.10.0-957.el7.x86_64
0xd1
20
? insert_kthread_work
vmlinux 0xffffffffa1ac1b60
Build id: 3.10.0-957.el7.x86_64
0x40
21
ret_from_fork_nospec_begin
vmlinux 0xffffffffa2174c37
Build id: 3.10.0-957.el7.x86_64
0x21
22
? insert_kthread_work
vmlinux 0xffffffffa1ac1b60
Build id: 3.10.0-957.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 21s! [ksoftirqd/7:44]
Modules linked in: tcp_diag inet_diag xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device
snd_pcm snd_timer snd joydev pcspkr soundcore sg i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm serio_raw e1000 ata_piix drm_panel_orientation_quirks libata mptspi scsi_transport_spi mptscsih mptbase nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 7 PID: 44 Comm: ksoftirqd/7 Kdump: loaded Tainted: G             L ------------   3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffff9564f994a080 ti: ffff9564f9960000 task.ti: ffff9564f9960000
RIP: 0010:[<ffffffffc00a93be>]  [<ffffffffc00a93be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff9564f9963aa0  EFLAGS: 00000246
RAX: ffffb978c1980000 RBX: ffff956714af5f00 RCX: 000000000000000c
RDX: ffff9567a9359600 RSI: ffff9567a9125000 RDI: 000000000000000e
RBP: ffff9564f9963ab0 R08: 0000000000000003 R09: 0000000282027fff
R10: ffff9567aedf10d8 R11: fffff943c067d500 R12: 0000000100000019
R13: ffff9565b2633900 R14: ffff9564f9963a14 R15: ffffffffa1d4b83b
FS:  0000000000000000(0000) GS:ffff9567ad7c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f980e9eb088 CR3: 00000001f34b6000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc00c471e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc010c2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa1ed0960>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa1ed9efc>] scsi_request_fn+0x4cc/0x680
[<ffffffffa1d42ff9>] __blk_run_queue+0x39/0x50
[<ffffffffa1d43076>] blk_run_queue+0x26/0x40
[<ffffffffa1ed84a8>] scsi_run_queue+0x258/0x2f0
[<ffffffffa1eda0f0>] scsi_next_command+0x20/0x40
[<ffffffffa1eda245>] scsi_end_request+0x135/0x1e0
[<ffffffffa1eda4b8>] scsi_io_completion+0x168/0x6a0
[<ffffffffa1ecf96c>] scsi_finish_command+0xdc/0x140
[<ffffffffa1ed9a02>] scsi_softirq_done+0x132/0x160
[<ffffffffa1d4cce6>] blk_done_softirq+0x96/0xc0
[<ffffffffa1aa0f05>] __do_softirq+0xf5/0x280
[<ffffffffa1aa10c8>] run_ksoftirqd+0x38/0x50
[<ffffffffa1aca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa1aca480>] ? lg_double_unlock+0x40/0x40
[<ffffffffa1ac1c31>] kthread+0xd1/0xe0
[<ffffffffa1ac1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2174c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa1ac1b60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 73 0b 
Complete report #1018058
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc030844e
Build id: 3.10.0-1160.108.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc036a74e
Build id: 3.10.0-1160.108.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03232f0
Build id: 3.10.0-1160.108.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffb8103160
Build id: 3.10.0-1160.108.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb810cd0c
Build id: 3.10.0-1160.108.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffb7f68799
Build id: 3.10.0-1160.108.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffb7f68826
Build id: 3.10.0-1160.108.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffb810b1e8
Build id: 3.10.0-1160.108.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffb810cf20
Build id: 3.10.0-1160.108.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffb810d08d
Build id: 3.10.0-1160.108.1.el7.x86_64
0x14d
11
scsi_io_completion
vmlinux 0xffffffffb810d310
Build id: 3.10.0-1160.108.1.el7.x86_64
0x170
12
scsi_finish_command
vmlinux 0xffffffffb81020d0
Build id: 3.10.0-1160.108.1.el7.x86_64
0xe0
13
scsi_softirq_done
vmlinux 0xffffffffb810c828
Build id: 3.10.0-1160.108.1.el7.x86_64
0x138
14
blk_done_softirq
vmlinux 0xffffffffb7f729c6
Build id: 3.10.0-1160.108.1.el7.x86_64
0x96
15
__do_softirq
vmlinux 0xffffffffb7ca9595
Build id: 3.10.0-1160.108.1.el7.x86_64
0xf5
16
run_ksoftirqd
vmlinux 0xffffffffb7ca9768
Build id: 3.10.0-1160.108.1.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffffb7cd43fc
Build id: 3.10.0-1160.108.1.el7.x86_64
0x14c
18
? lg_double_unlock
vmlinux 0xffffffffb7cd42b0
Build id: 3.10.0-1160.108.1.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffffb7ccb621
Build id: 3.10.0-1160.108.1.el7.x86_64
0xd1
20
? insert_kthread_work
vmlinux 0xffffffffb7ccb550
Build id: 3.10.0-1160.108.1.el7.x86_64
0x40
21
ret_from_fork_nospec_begin
vmlinux 0xffffffffb83c51dd
Build id: 3.10.0-1160.108.1.el7.x86_64
0x7
22
? insert_kthread_work
vmlinux 0xffffffffb7ccb550
Build id: 3.10.0-1160.108.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [ksoftirqd/11:67]
Modules linked in: xt_owner xt_REDIRECT nf_nat_redirect xt_multiport ipt_rpfilter iptable_raw ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel rpcsec_gss_krb5 veth auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache xt_set ip_set_hash_ipportnet ip_set_bitmap_port ip_set_hash_ipportip ip_set_hash_ipport dummy nf_tables ip_set ip_vs_sh ip_vs_wrr ip_vs_rr ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_filter ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6_tables xt_mark xt_comment iptable_mangle xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat xt_addrtype iptable_filter scini(POE) ip_vs nf_conntrack binfmt_misc vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon
sg parport_pc joydev parport pcspkr vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix mptspi scsi_transport_spi mptscsih drm libata crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod br_netfilter bridge stp llc overlay(T)
CPU: 11 PID: 67 Comm: ksoftirqd/11 Tainted: P        W  OEL ------------ T 3.10.0-1160.108.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffffa05f7d559080 ti: ffffa05f7d564000 task.ti: ffffa05f7d564000
RIP: 0010:[<ffffffffc030844e>]  [<ffffffffc030844e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
IPVS: rr: TCP 10.104.197.84:9200 - no destination available
PowerFlex netCon_MitEntry_CK:1618 :Mit: ffffa05d750fd718. Chan ffffb2c3c241c010, Con ffffa05d750fd640 ConnectToPeer returned 4(NOT_CONN)
RSP: 0018:ffffa05f7d567aa0  EFLAGS: 00000246
RAX: ffffb2c3c1c00000 RBX: ffffa05f02bde520 RCX: 0000000000000004
RDX: ffffa05df7b58c00 RSI: ffffa05efaffb000 RDI: 000000000000000e
RBP: ffffa05f7d567ab0 R08: 0000000000000003 R09: 0000000226d28fff
R10: 0000000055415f01 R11: ffffa05f55415180 R12: 0000000100000002
R13: ffffa05feb7c2c00 R14: ffffa05f7d567a14 R15: ffffffffb7f7154b
FS:  0000000000000000(0000) GS:ffffa05fffd80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005638efe2b000 CR3: 000000003ee10000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc036a74e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03232f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb8103160>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffffb810cd0c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb7f68799>] __blk_run_queue+0x39/0x50
[<ffffffffb7f68826>] blk_run_queue+0x26/0x40
[<ffffffffb810b1e8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb810cf20>] scsi_next_command+0x20/0x40
[<ffffffffb810d08d>] scsi_end_request+0x14d/0x200
[<ffffffffb810d310>] scsi_io_completion+0x170/0x720
[<ffffffffb81020d0>] scsi_finish_command+0xe0/0x140
[<ffffffffb810c828>] scsi_softirq_done+0x138/0x170
[<ffffffffb7f729c6>] blk_done_softirq+0x96/0xd0
[<ffffffffb7ca9595>] __do_softirq+0xf5/0x290
[<ffffffffb7ca9768>] run_ksoftirqd+0x38/0x60
[<ffffffffb7cd43fc>] smpboot_thread_fn+0x14c/0x1b0
[<ffffffffb7cd42b0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb7ccb621>] kthread+0xd1/0xe0
[<ffffffffb7ccb550>] ? insert_kthread_work+0x40/0x40
[<ffffffffb83c51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffb7ccb550>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 
Complete report #1008790
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc02b344e
Build id: 3.10.0-1160.114.2.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc02ce74e
Build id: 3.10.0-1160.114.2.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc02912f0
Build id: 3.10.0-1160.114.2.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffb5b03150
Build id: 3.10.0-1160.114.2.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb5b0ccfc
Build id: 3.10.0-1160.114.2.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffb59687b9
Build id: 3.10.0-1160.114.2.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffb5968846
Build id: 3.10.0-1160.114.2.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffb5b0b1d8
Build id: 3.10.0-1160.114.2.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffb5b0cf10
Build id: 3.10.0-1160.114.2.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffb5b0d07d
Build id: 3.10.0-1160.114.2.el7.x86_64
0x14d
11
scsi_io_completion
vmlinux 0xffffffffb5b0d300
Build id: 3.10.0-1160.114.2.el7.x86_64
0x170
12
? __enqueue_entity
vmlinux 0xffffffffb56e5d58
Build id: 3.10.0-1160.114.2.el7.x86_64
0x78
13
scsi_finish_command
vmlinux 0xffffffffb5b020c0
Build id: 3.10.0-1160.114.2.el7.x86_64
0xe0
14
scsi_softirq_done
vmlinux 0xffffffffb5b0c818
Build id: 3.10.0-1160.114.2.el7.x86_64
0x138
15
blk_done_softirq
vmlinux 0xffffffffb59729e6
Build id: 3.10.0-1160.114.2.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffffb56a9595
Build id: 3.10.0-1160.114.2.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffffb56a9768
Build id: 3.10.0-1160.114.2.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffffb56d43fc
Build id: 3.10.0-1160.114.2.el7.x86_64
0x14c
19
? lg_double_unlock
vmlinux 0xffffffffb56d42b0
Build id: 3.10.0-1160.114.2.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffffb56cb621
Build id: 3.10.0-1160.114.2.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffffb56cb550
Build id: 3.10.0-1160.114.2.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffffb5dc51f7
Build id: 3.10.0-1160.114.2.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffffb56cb550
Build id: 3.10.0-1160.114.2.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ksoftirqd/6:39]
Modules linked in: traps(OE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nf_log_ipv4 nf_log_common xt_LOG ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc squashfs loop sb_edac iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev vmw_balloon sg parport_pc parport vmw_vmci i2c_piix4 binfmt_misc ip_tables
xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit crct10dif_pclmul ata_piix crct10dif_common drm serio_raw crc32c_intel libnvdimm libata mptspi scsi_transport_spi mptscsih mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 6 PID: 39 Comm: ksoftirqd/6 Kdump: loaded Tainted: G           OE  ------------   3.10.0-1160.114.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff965299693180 ti: ffff9652996a0000 task.ti: ffff9652996a0000
RIP: 0010:[<ffffffffc02b344e>]  [<ffffffffc02b344e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
RSP: 0018:ffff9652996a3aa0  EFLAGS: 00000246
RAX: ffffa8e886ac0000 RBX: ffff9660f778d120 RCX: 000000000000004b
RDX: ffff965a403d1200 RSI: ffff96613a282000 RDI: 000000000000000e
RBP: ffff9652996a3ab0 R08: 0000000000000003 R09: 0000000fcb994fff
R10: ffff96617fda3cf0 R11: fffff646badaad00 R12: 0000000100000002
R13: ffff965fdc598a80 R14: ffff9652996a3a14 R15: ffffffffb597156b
FS:  0000000000000000(0000) GS:ffff96617fd80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fec76197000 CR3: 0000000ff6dbe000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc02ce74e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02912f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb5b03150>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffffb5b0ccfc>] scsi_request_fn+0x4ac/0x680
[<ffffffffb59687b9>] __blk_run_queue+0x39/0x50
[<ffffffffb5968846>] blk_run_queue+0x26/0x40
[<ffffffffb5b0b1d8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb5b0cf10>] scsi_next_command+0x20/0x40
[<ffffffffb5b0d07d>] scsi_end_request+0x14d/0x200
[<ffffffffb5b0d300>] scsi_io_completion+0x170/0x720
[<ffffffffb56e5d58>] ? __enqueue_entity+0x78/0x80
[<ffffffffb5b020c0>] scsi_finish_command+0xe0/0x140
[<ffffffffb5b0c818>] scsi_softirq_done+0x138/0x170
[<ffffffffb59729e6>] blk_done_softirq+0x96/0xd0
[<ffffffffb56a9595>] __do_softirq+0xf5/0x290
[<ffffffffb56a9768>] run_ksoftirqd+0x38/0x60
[<ffffffffb56d43fc>] smpboot_thread_fn+0x14c/0x1b0
[<ffffffffb56d42b0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb56cb621>] kthread+0xd1/0xe0
[<ffffffffb56cb550>] ? insert_kthread_work+0x40/0x40
[<ffffffffb5dc51f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb56cb550>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 
Complete report #688206
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03983be
Build id: 3.10.0-957.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc037a71e
Build id: 3.10.0-957.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03c22e0
Build id: 3.10.0-957.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff818d0960
Build id: 3.10.0-957.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff818d9efc
Build id: 3.10.0-957.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffff81742ff9
Build id: 3.10.0-957.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff81743076
Build id: 3.10.0-957.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff818d84a8
Build id: 3.10.0-957.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff818da0f0
Build id: 3.10.0-957.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff818da245
Build id: 3.10.0-957.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff818da4b8
Build id: 3.10.0-957.el7.x86_64
0x168
12
? __schedule
vmlinux 0xffffffff81b678fa
Build id: 3.10.0-957.el7.x86_64
0x5ea
13
scsi_finish_command
vmlinux 0xffffffff818cf96c
Build id: 3.10.0-957.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffff818d9a02
Build id: 3.10.0-957.el7.x86_64
0x132
15
blk_done_softirq
vmlinux 0xffffffff8174cce6
Build id: 3.10.0-957.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffff814a0f05
Build id: 3.10.0-957.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffff814a10c8
Build id: 3.10.0-957.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff814ca5c4
Build id: 3.10.0-957.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffff814ca480
Build id: 3.10.0-957.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffff814c1c31
Build id: 3.10.0-957.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffff814c1b60
Build id: 3.10.0-957.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffff81b74c24
Build id: 3.10.0-957.el7.x86_64
0xe
23
? insert_kthread_work
vmlinux 0xffffffff814c1b60
Build id: 3.10.0-957.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:3]
Modules linked in: rfcomm bnep btusb btrtl btbcm btintel bluetooth rfkill tcp_lp nls_utf8 isofs fuse ebtable_filter ebtables devlink target_core_user uio target_core_pscsi target_core_file nf_conntrack_netlink target_core_iblock nfnetlink iscsi_target_mod target_core_mod iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi openvswitch nf_nat_ipv6 ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables vmw_vsock_vmci_transport vsock xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_comment xt_conntrack nf_conntrack xt_multiport iptable_filter sunrpc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon
aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 joydev loop pcspkr snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore sg parport_pc parport vmw_vmci i2c_piix4 br_netfilter bridge stp llc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm e1000 mptspi scsi_transport_spi mptscsih drm_panel_orientation_quirks mptbase nfit libnvdimm ata_generic pata_acpi ata_piix libata dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff8f26fcf72080 ti: ffff8f26fc800000 task.ti: ffff8f26fc800000
RIP: 0010:[<ffffffffc03983be>]  [<ffffffffc03983be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8f26fc803aa0  EFLAGS: 00000246
RAX: ffffac8940cc0000 RBX: ffff8f26ac0241a0 RCX: 0000000000000003
RDX: ffff8f27554c4400 RSI: ffff8f25b5461000 RDI: 000000000000000e
RBP: ffff8f26fc803ab0 R08: 0000000000000003 R09: 0000000150246fff
R10: ffff8f25b54630d8 R11: fffff58088ffa400 R12: 000000010000000e
R13: ffff8f2589bf4000 R14: ffff8f26fc803a14 R15: ffffffff8174b83b
FS:  0000000000000000(0000) GS:ffff8f27b5600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f76e5a2d000 CR3: 00000001c2f78000 CR4: 00000000003406f0
Call Trace:
[<ffffffffc037a71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03c22e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff818d0960>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff818d9efc>] scsi_request_fn+0x4cc/0x680
[<ffffffff81742ff9>] __blk_run_queue+0x39/0x50
[<ffffffff81743076>] blk_run_queue+0x26/0x40
[<ffffffff818d84a8>] scsi_run_queue+0x258/0x2f0
[<ffffffff818da0f0>] scsi_next_command+0x20/0x40
[<ffffffff818da245>] scsi_end_request+0x135/0x1e0
[<ffffffff818da4b8>] scsi_io_completion+0x168/0x6a0
[<ffffffff81b678fa>] ? __schedule+0x5ea/0x890
[<ffffffff818cf96c>] scsi_finish_command+0xdc/0x140
[<ffffffff818d9a02>] scsi_softirq_done+0x132/0x160
[<ffffffff8174cce6>] blk_done_softirq+0x96/0xc0
[<ffffffff814a0f05>] __do_softirq+0xf5/0x280
[<ffffffff814a10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff814ca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff814ca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff814c1c31>] kthread+0xd1/0xe0
[<ffffffff814c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff81b74c24>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffff814c1b60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 63 3a 
Complete report #657707
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01653be
Build id: 3.10.0-1160.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc018071e
Build id: 3.10.0-1160.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc02162e0
Build id: 3.10.0-1160.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa26e3980
Build id: 3.10.0-1160.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffa26ed0ac
Build id: 3.10.0-1160.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffa25534d9
Build id: 3.10.0-1160.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffa2553556
Build id: 3.10.0-1160.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffa26eb608
Build id: 3.10.0-1160.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffa26ed2c0
Build id: 3.10.0-1160.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffa26ed415
Build id: 3.10.0-1160.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffa26ed688
Build id: 3.10.0-1160.el7.x86_64
0x168
12
? scsi_io_completion
vmlinux 0xffffffffa26ed668
Build id: 3.10.0-1160.el7.x86_64
0x148
13
scsi_finish_command
vmlinux 0xffffffffa26e297c
Build id: 3.10.0-1160.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffffa26ecbd0
Build id: 3.10.0-1160.el7.x86_64
0x130
15
blk_done_softirq
vmlinux 0xffffffffa255d226
Build id: 3.10.0-1160.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffffa22a4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffffa22a4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffffa22ce674
Build id: 3.10.0-1160.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffffa22ce530
Build id: 3.10.0-1160.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffffa22c5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffffa22c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffffa2993de4
Build id: 3.10.0-1160.el7.x86_64
0xe
23
? insert_kthread_work
vmlinux 0xffffffffa22c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:6]
Modules linked in: xt_nat ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs xt_comment xt_mark nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper snd_timer cryptd snd ppdev soundcore joydev vmw_balloon pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables
xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel nfit libnvdimm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm serio_raw drm ata_piix mptspi e1000 libata scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffffa0a1773b5280 ti: ffffa0a1777f4000 task.ti: ffffa0a1777f4000
RIP: 0010:[<ffffffffc01653be>]  [<ffffffffc01653be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffffa0a1777f7aa0  EFLAGS: 00000246
RAX: ffffaec7c0400000 RBX: ffffa0a172878000 RCX: 0000000000000009
RDX: ffffa0a176db3200 RSI: ffffa0a1358f2000 RDI: 000000000000000e
RBP: ffffa0a1777f7ab0 R08: 0000000000000003 R09: 000000001ba66fff
R10: 00000000358bf801 R11: ffffa0a1358bd500 R12: 00000000005b8d80
R13: 00000000777f7a58 R14: ffffffffa22e54ea R15: ffffa0a1777f7a40
FS:  00007ff70bde8a40(0000) GS:ffffa0a17b600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff70b30ad00 CR3: 0000000004dc0000 CR4: 00000000003407f0
Call Trace:
[<ffffffffc018071e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02162e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa26e3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa26ed0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffffa25534d9>] __blk_run_queue+0x39/0x50
[<ffffffffa2553556>] blk_run_queue+0x26/0x40
[<ffffffffa26eb608>] scsi_run_queue+0x258/0x2f0
[<ffffffffa26ed2c0>] scsi_next_command+0x20/0x40
[<ffffffffa26ed415>] scsi_end_request+0x135/0x1e0
[<ffffffffa26ed688>] scsi_io_completion+0x168/0x720
[<ffffffffa26ed668>] ? scsi_io_completion+0x148/0x720
[<ffffffffa26e297c>] scsi_finish_command+0xdc/0x140
[<ffffffffa26ecbd0>] scsi_softirq_done+0x130/0x160
[<ffffffffa255d226>] blk_done_softirq+0x96/0xc0
[<ffffffffa22a4b95>] __do_softirq+0xf5/0x280
[<ffffffffa22a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffa22ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa22ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffffa22c5c21>] kthread+0xd1/0xe0
[<ffffffffa22c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2993de4>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffffa22c5b50>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 33 17 
Complete report #166717
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff939883c5
Build id: 3.10.0-1127.19.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0432632
Build id: 3.10.0-1127.19.1.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff936eb0a0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff936e2990
Build id: 3.10.0-1127.19.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff936ec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffff935527a9
Build id: 3.10.0-1127.19.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff93552826
Build id: 3.10.0-1127.19.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff936ea618
Build id: 3.10.0-1127.19.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff936ec2d0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff936ec425
Build id: 3.10.0-1127.19.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff936ec698
Build id: 3.10.0-1127.19.1.el7.x86_64
0x168
12
? __netif_receive_skb
vmlinux 0xffffffff93853f18
Build id: 3.10.0-1127.19.1.el7.x86_64
0x18
13
scsi_finish_command
vmlinux 0xffffffff936e198c
Build id: 3.10.0-1127.19.1.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffff936ebbe0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x130
15
blk_done_softirq
vmlinux 0xffffffff9355c4f6
Build id: 3.10.0-1127.19.1.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffff932a5695
Build id: 3.10.0-1127.19.1.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffff932a5858
Build id: 3.10.0-1127.19.1.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff932cf0e4
Build id: 3.10.0-1127.19.1.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffff932cefa0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffff932c6691
Build id: 3.10.0-1127.19.1.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffff932c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffff93992d37
Build id: 3.10.0-1127.19.1.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffff932c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink ipmi_devintf ipmi_msghandler vmw_vsock_vmci_transport vsock sunrpc sb_edac ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper sg cryptd vmw_balloon pcspkr joydev vmxnet3 parport_pc parport i2c_piix4 vmw_vmci binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif drm_kms_helper crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops
ttm crct10dif_pclmul crct10dif_common drm crc32c_intel serio_raw ata_piix libata nfit vmw_pvscsi libnvdimm drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffffa050bb315230 ti: ffffa050bb34c000 task.ti: ffffa050bb34c000
RIP: 0010:[<ffffffff939883c5>]  [<ffffffff939883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffffa050bb34fb08  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffa050bb34fac8 RCX: ffffa050badbd060
RDX: ffffaf0541b80000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffffa050bb34fb08 R08: 0000000000000000 R09: ffffa051d0a2cb00
R10: ffffa0526ebe5400 R11: ffffa0526eb4a800 R12: 0000000000000020
R13: ffffa04ffab38900 R14: ffffffff936e9d59 R15: ffffa050bb34fa80
FS:  0000000000000000(0000) GS:ffffa0527fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f743e369dc5 CR3: 00000001a7bc0000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc0432632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff936eb0a0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff936e2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff936ec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff935527a9>] __blk_run_queue+0x39/0x50
[<ffffffff93552826>] blk_run_queue+0x26/0x40
[<ffffffff936ea618>] scsi_run_queue+0x258/0x2f0
[<ffffffff936ec2d0>] scsi_next_command+0x20/0x40
[<ffffffff936ec425>] scsi_end_request+0x135/0x1e0
[<ffffffff936ec698>] scsi_io_completion+0x168/0x720
[<ffffffff93853f18>] ? __netif_receive_skb+0x18/0x60
[<ffffffff936e198c>] scsi_finish_command+0xdc/0x140
[<ffffffff936ebbe0>] scsi_softirq_done+0x130/0x160
[<ffffffff9355c4f6>] blk_done_softirq+0x96/0xc0
[<ffffffff932a5695>] __do_softirq+0xf5/0x280
[<ffffffff932a5858>] run_ksoftirqd+0x38/0x50
[<ffffffff932cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff932cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffff932c6691>] kthread+0xd1/0xe0
[<ffffffff932c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff93992d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff932c65c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [CRTC:37:crtc-0] flip_done timed out
Complete report #519270
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03223be
Build id: 3.10.0-1160.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc03aa71e
Build id: 3.10.0-1160.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03a32e0
Build id: 3.10.0-1160.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff9fce3980
Build id: 3.10.0-1160.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff9fced0ac
Build id: 3.10.0-1160.el7.x86_64
0x4ac
6
? kmem_cache_free
vmlinux 0xffffffff9fa26642
Build id: 3.10.0-1160.el7.x86_64
0x1e2
7
__blk_run_queue
vmlinux 0xffffffff9fb534d9
Build id: 3.10.0-1160.el7.x86_64
0x39
8
blk_run_queue
vmlinux 0xffffffff9fb53556
Build id: 3.10.0-1160.el7.x86_64
0x26
9
scsi_run_queue
vmlinux 0xffffffff9fceb608
Build id: 3.10.0-1160.el7.x86_64
0x258
10
scsi_next_command
vmlinux 0xffffffff9fced2c0
Build id: 3.10.0-1160.el7.x86_64
0x20
11
scsi_end_request
vmlinux 0xffffffff9fced415
Build id: 3.10.0-1160.el7.x86_64
0x135
12
scsi_io_completion
vmlinux 0xffffffff9fced688
Build id: 3.10.0-1160.el7.x86_64
0x168
13
scsi_finish_command
vmlinux 0xffffffff9fce297c
Build id: 3.10.0-1160.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffff9fcecbd0
Build id: 3.10.0-1160.el7.x86_64
0x130
15
blk_done_softirq
vmlinux 0xffffffff9fb5d226
Build id: 3.10.0-1160.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffff9f8a4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffff9f8a4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff9f8ce674
Build id: 3.10.0-1160.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffff9f8ce530
Build id: 3.10.0-1160.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffff9f8c5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffff9f8c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffff9ff93df7
Build id: 3.10.0-1160.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffff9f8c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ksoftirqd/1:14]
Modules linked in: tcp_lp nls_utf8 isofs xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod xfs libcrc32c iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul joydev pcspkr glue_helper vmw_balloon ablk_helper cryptd ppdev sg parport_pc parport snd_seq_midi snd_seq_midi_event
snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore i2c_piix4 vmw_vmci ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix mptspi drm nfit scsi_transport_spi libata mptscsih libnvdimm serio_raw mptbase e1000 drm_panel_orientation_quirks fuse
CPU: 1 PID: 14 Comm: ksoftirqd/1 Tainted: G             L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffff8e66ffb18000 ti: ffff8e66ffb20000 task.ti: ffff8e66ffb20000
RIP: 0010:[<ffffffffc03223be>]  [<ffffffffc03223be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff8e66ffb23aa0  EFLAGS: 00000246
RAX: ffff99b8c0240000 RBX: ffff8e66fc65acc0 RCX: 0000000000000004
RDX: ffff8e66f5ebb400 RSI: ffff8e66f5754000 RDI: 000000000000000e
RBP: ffff8e66ffb23ab0 R08: 0000000000000003 R09: 000000002cc1cfff
R10: ffff8e66fc663c70 R11: ffffd7a980dc6b00 R12: ffffffff9f8d7229
R13: ffff8e66ffb23a30 R14: ffff8e66d6a51080 R15: ffffffff9f8d7202
FS:  0000000000000000(0000) GS:ffff8e66fc640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000405ed0 CR3: 0000000036fc0000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc03aa71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03a32e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9fce3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9fced0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffff9fa26642>] ? kmem_cache_free+0x1e2/0x200
[<ffffffff9fb534d9>] __blk_run_queue+0x39/0x50
[<ffffffff9fb53556>] blk_run_queue+0x26/0x40
[<ffffffff9fceb608>] scsi_run_queue+0x258/0x2f0
[<ffffffff9fced2c0>] scsi_next_command+0x20/0x40
[<ffffffff9fced415>] scsi_end_request+0x135/0x1e0
[<ffffffff9fced688>] scsi_io_completion+0x168/0x720
[<ffffffff9fce297c>] scsi_finish_command+0xdc/0x140
[<ffffffff9fcecbd0>] scsi_softirq_done+0x130/0x160
[<ffffffff9fb5d226>] blk_done_softirq+0x96/0xc0
[<ffffffff9f8a4b95>] __do_softirq+0xf5/0x280
[<ffffffff9f8a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff9f8ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9f8ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffff9f8c5c21>] kthread+0xd1/0xe0
[<ffffffff9f8c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff9ff93df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9f8c5b50>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 03 33 
Complete report #541994
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01ad3be
Build id: 3.10.0-1127.18.2.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc023871e
Build id: 3.10.0-1127.18.2.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc01a52e0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffaf8e2950
Build id: 3.10.0-1127.18.2.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffaf8ec07c
Build id: 3.10.0-1127.18.2.el7.x86_64
0x4ac
6
? kmem_cache_free
vmlinux 0xffffffffaf626702
Build id: 3.10.0-1127.18.2.el7.x86_64
0x1e2
7
__blk_run_queue
vmlinux 0xffffffffaf752769
Build id: 3.10.0-1127.18.2.el7.x86_64
0x39
8
blk_run_queue
vmlinux 0xffffffffaf7527e6
Build id: 3.10.0-1127.18.2.el7.x86_64
0x26
9
scsi_run_queue
vmlinux 0xffffffffaf8ea5d8
Build id: 3.10.0-1127.18.2.el7.x86_64
0x258
10
scsi_next_command
vmlinux 0xffffffffaf8ec290
Build id: 3.10.0-1127.18.2.el7.x86_64
0x20
11
scsi_end_request
vmlinux 0xffffffffaf8ec3e5
Build id: 3.10.0-1127.18.2.el7.x86_64
0x135
12
scsi_io_completion
vmlinux 0xffffffffaf8ec658
Build id: 3.10.0-1127.18.2.el7.x86_64
0x168
13
? put_prev_entity
vmlinux 0xffffffffaf4e69fb
Build id: 3.10.0-1127.18.2.el7.x86_64
0x8b
14
scsi_finish_command
vmlinux 0xffffffffaf8e194c
Build id: 3.10.0-1127.18.2.el7.x86_64
0xdc
15
scsi_softirq_done
vmlinux 0xffffffffaf8ebba0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x130
16
blk_done_softirq
vmlinux 0xffffffffaf75c4b6
Build id: 3.10.0-1127.18.2.el7.x86_64
0x96
17
__do_softirq
vmlinux 0xffffffffaf4a5695
Build id: 3.10.0-1127.18.2.el7.x86_64
0xf5
18
run_ksoftirqd
vmlinux 0xffffffffaf4a5858
Build id: 3.10.0-1127.18.2.el7.x86_64
0x38
19
smpboot_thread_fn
vmlinux 0xffffffffaf4cf0e4
Build id: 3.10.0-1127.18.2.el7.x86_64
0x144
20
? lg_double_unlock
vmlinux 0xffffffffaf4cefa0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x40
21
kthread
vmlinux 0xffffffffaf4c6691
Build id: 3.10.0-1127.18.2.el7.x86_64
0xd1
22
? insert_kthread_work
vmlinux 0xffffffffaf4c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x40
23
ret_from_fork_nospec_begin
vmlinux 0xffffffffafb92d37
Build id: 3.10.0-1127.18.2.el7.x86_64
0x21
24
? insert_kthread_work
vmlinux 0xffffffffaf4c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 29s! [ksoftirqd/9:54]
Modules linked in: ip6table_mangle ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_filter iptable_mangle ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables xt_comment xt_mark veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_nat_ipv4 nf_nat br_netfilter bridge stp llc binfmt_misc vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon joydev pcspkr sg parport_pc i2c_piix4 parport vmw_vmci overlay(T) nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack ip_tables xfs dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit drm libnvdimm ata_piix mptspi libata scsi_transport_spi mptscsih crct10dif_pclmul crct10dif_common crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ipmi_msghandler]
CPU: 9 PID: 54 Comm: ksoftirqd/9 Tainted: G             L ------------ T 3.10.0-1127.18.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/19/2018
task: ffff93eae9735230 ti: ffff93eae9750000 task.ti: ffff93eae9750000
RIP: 0010:[<ffffffffc01ad3be>]  [<ffffffffc01ad3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff93eae9753aa0  EFLAGS: 00000246
RAX: ffffaee1c6a40000 RBX: ffff93f4d8a3ee00 RCX: 000000000000000f
RDX: ffff93f95cc92400 RSI: ffff93f979469000 RDI: 000000000000000e
RBP: ffff93eae9753ab0 R08: 0000000000000002 R09: 0000000dc5860fff
R10: ffff93f978bfe000 R11: ffff93eb3a793800 R12: 0000000100000001
R13: ffff93ea91bb0300 R14: ffff93eae9753a14 R15: ffffffffaf75b08b
FS:  0000000000000000(0000) GS:ffff93f97f440000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f126493e000 CR3: 0000000fdc610000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc023871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01a52e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffaf8e2950>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffaf8ec07c>] scsi_request_fn+0x4ac/0x680
[<ffffffffaf626702>] ? kmem_cache_free+0x1e2/0x200
[<ffffffffaf752769>] __blk_run_queue+0x39/0x50
[<ffffffffaf7527e6>] blk_run_queue+0x26/0x40
[<ffffffffaf8ea5d8>] scsi_run_queue+0x258/0x2f0
[<ffffffffaf8ec290>] scsi_next_command+0x20/0x40
[<ffffffffaf8ec3e5>] scsi_end_request+0x135/0x1e0
[<ffffffffaf8ec658>] scsi_io_completion+0x168/0x720
[<ffffffffaf4e69fb>] ? put_prev_entity+0x8b/0x400
[<ffffffffaf8e194c>] scsi_finish_command+0xdc/0x140
[<ffffffffaf8ebba0>] scsi_softirq_done+0x130/0x160
[<ffffffffaf75c4b6>] blk_done_softirq+0x96/0xc0
[<ffffffffaf4a5695>] __do_softirq+0xf5/0x280
[<ffffffffaf4a5858>] run_ksoftirqd+0x38/0x50
[<ffffffffaf4cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffaf4cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffaf4c6691>] kthread+0xd1/0xe0
[<ffffffffaf4c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffafb92d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffaf4c65c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 1b 
Complete report #503590
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc04ab3be
Build id: 3.10.0-1062.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc046f71e
Build id: 3.10.0-1062.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc04152e0
Build id: 3.10.0-1062.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffaa0dc890
Build id: 3.10.0-1062.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffaa0e5efc
Build id: 3.10.0-1062.el7.x86_64
0x4cc
6
? try_to_wake_up
vmlinux 0xffffffffa9cd9df0
Build id: 3.10.0-1062.el7.x86_64
0x190
7
__blk_run_queue
vmlinux 0xffffffffa9f4cf59
Build id: 3.10.0-1062.el7.x86_64
0x39
8
blk_run_queue
vmlinux 0xffffffffa9f4cfd6
Build id: 3.10.0-1062.el7.x86_64
0x26
9
scsi_run_queue
vmlinux 0xffffffffaa0e44a8
Build id: 3.10.0-1062.el7.x86_64
0x258
10
scsi_next_command
vmlinux 0xffffffffaa0e60f0
Build id: 3.10.0-1062.el7.x86_64
0x20
11
scsi_end_request
vmlinux 0xffffffffaa0e6245
Build id: 3.10.0-1062.el7.x86_64
0x135
12
scsi_io_completion
vmlinux 0xffffffffaa0e6476
Build id: 3.10.0-1062.el7.x86_64
0x126
13
scsi_finish_command
vmlinux 0xffffffffaa0db89c
Build id: 3.10.0-1062.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffffaa0e5a02
Build id: 3.10.0-1062.el7.x86_64
0x132
15
blk_done_softirq
vmlinux 0xffffffffa9f56c56
Build id: 3.10.0-1062.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffffa9ca41e5
Build id: 3.10.0-1062.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffffa9ca43a8
Build id: 3.10.0-1062.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffffa9ccdb04
Build id: 3.10.0-1062.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffffa9ccd9c0
Build id: 3.10.0-1062.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffffa9cc50d1
Build id: 3.10.0-1062.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffffa9cc5000
Build id: 3.10.0-1062.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffffaa38bd37
Build id: 3.10.0-1062.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffffa9cc5000
Build id: 3.10.0-1062.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: xt_recent veth nf_tables xt_statistic ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 rpcsec_gss_krb5 nfsv4 dns_resolver nfs lockd grace fscache nf_conntrack_netlink xt_nat xt_mark xt_addrtype xt_comment ip_set nfnetlink ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio sb_edac iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg joydev parport_pc i2c_piix4 parport
vmw_vmci br_netfilter bridge auth_rpcgss stp sunrpc llc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi ata_piix scsi_transport_spi mptscsih libata crct10dif_pclmul mptbase crct10dif_common crc32c_intel serio_raw vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff97c5fa28d230 ti: ffff97c5fa2f8000 task.ti: ffff97c5fa2f8000
RIP: 0010:[<ffffffffc04ab3be>]  [<ffffffffc04ab3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff97c5fa2fbaa0  EFLAGS: 00000246
RAX: ffffbc2d83600000 RBX: ffff97c7c9e74900 RCX: 000000000000001f
RDX: ffff97ccace53c00 RSI: ffff97c8adbb7000 RDI: 000000000000000e
RBP: ffff97c5fa2fbab0 R08: 0000000000000002 R09: 000000026343ffff
R10: ffff97c5f9d3c0d8 R11: ffff97c843f46bc0 R12: 0000000100000001
R13: ffff97c6737bad00 R14: ffff97c5fa2fba14 R15: ffffffffa9f557ab
FS:  0000000000000000(0000) GS:ffff97c8afc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000560a0a27e010 CR3: 0000000145410000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc046f71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04152e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffaa0dc890>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffaa0e5efc>] scsi_request_fn+0x4cc/0x680
[<ffffffffa9cd9df0>] ? try_to_wake_up+0x190/0x390
[<ffffffffa9f4cf59>] __blk_run_queue+0x39/0x50
[<ffffffffa9f4cfd6>] blk_run_queue+0x26/0x40
[<ffffffffaa0e44a8>] scsi_run_queue+0x258/0x2f0
[<ffffffffaa0e60f0>] scsi_next_command+0x20/0x40
[<ffffffffaa0e6245>] scsi_end_request+0x135/0x1e0
[<ffffffffaa0e6476>] scsi_io_completion+0x126/0x6a0
[<ffffffffaa0db89c>] scsi_finish_command+0xdc/0x140
[<ffffffffaa0e5a02>] scsi_softirq_done+0x132/0x160
[<ffffffffa9f56c56>] blk_done_softirq+0x96/0xc0
[<ffffffffa9ca41e5>] __do_softirq+0xf5/0x280
[<ffffffffa9ca43a8>] run_ksoftirqd+0x38/0x50
[<ffffffffa9ccdb04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa9ccd9c0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa9cc50d1>] kthread+0xd1/0xe0
[<ffffffffa9cc5000>] ? insert_kthread_work+0x40/0x40
[<ffffffffaa38bd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa9cc5000>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 93 4b 
Complete report #425550
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff8d71b2c5
Build id: 3.10.0-862.11.6.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc02cd622
Build id: 3.10.0-862.11.6.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff8d4a8500
Build id: 3.10.0-862.11.6.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff8d4a0010
Build id: 3.10.0-862.11.6.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff8d4a957c
Build id: 3.10.0-862.11.6.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffff8d31d4e9
Build id: 3.10.0-862.11.6.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff8d31d566
Build id: 3.10.0-862.11.6.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff8d4a7b18
Build id: 3.10.0-862.11.6.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff8d4a9770
Build id: 3.10.0-862.11.6.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff8d4a98c5
Build id: 3.10.0-862.11.6.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff8d4a9b38
Build id: 3.10.0-862.11.6.el7.x86_64
0x168
12
scsi_finish_command
vmlinux 0xffffffff8d49f01c
Build id: 3.10.0-862.11.6.el7.x86_64
0xdc
13
scsi_softirq_done
vmlinux 0xffffffff8d4a9082
Build id: 3.10.0-862.11.6.el7.x86_64
0x132
14
blk_done_softirq
vmlinux 0xffffffff8d3271a6
Build id: 3.10.0-862.11.6.el7.x86_64
0x96
15
__do_softirq
vmlinux 0xffffffff8d09dba5
Build id: 3.10.0-862.11.6.el7.x86_64
0xf5
16
run_ksoftirqd
vmlinux 0xffffffff8d09dd68
Build id: 3.10.0-862.11.6.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffff8d0c6744
Build id: 3.10.0-862.11.6.el7.x86_64
0x144
18
? lg_double_unlock
vmlinux 0xffffffff8d0c6600
Build id: 3.10.0-862.11.6.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffff8d0bdf21
Build id: 3.10.0-862.11.6.el7.x86_64
0xd1
20
? insert_kthread_work
vmlinux 0xffffffff8d0bde50
Build id: 3.10.0-862.11.6.el7.x86_64
0x40
21
ret_from_fork_nospec_begin
vmlinux 0xffffffff8d7255dd
Build id: 3.10.0-862.11.6.el7.x86_64
0x7
22
? insert_kthread_work
vmlinux 0xffffffff8d0bde50
Build id: 3.10.0-862.11.6.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 26s! [ksoftirqd/0:3]
Modules linked in: veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter binfmt_misc gsch(OE) redirfs(OE) event_collector_1_9_8557(POE) ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon
ppdev pcspkr sg joydev parport_pc shpchp parport i2c_piix4 vmw_vmci auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata crct10dif_pclmul crct10dif_common i2c_core crc32c_intel serio_raw vmxnet3 vmw_pvscsi dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: P        W  OEL ------------   3.10.0-862.11.6.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9755ecc80000 ti: ffff9755ecc88000 task.ti: ffff9755ecc88000
RIP: 0010:[<ffffffff8d71b2c5>]  [<ffffffff8d71b2c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9755ecc8bb08  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff9755ecc8bac8 RCX: ffff9763cee70060
RDX: ffffa4044ce20000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff9755ecc8bb08 R08: 0000000000000000 R09: ffff9760b0f7e000
R10: ffff97610b09e8d8 R11: ffff97610b041800 R12: 0000000000000020
R13: ffff976cd7a9ef00 R14: ffffffff8d4a7259 R15: ffff9755ecc8ba80
FS:  0000000000000000(0000) GS:ffff97610d800000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000448e20 CR3: 0000000f4e80e000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc02cd622>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8d4a8500>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8d4a0010>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8d4a957c>] scsi_request_fn+0x4cc/0x680
[<ffffffff8d31d4e9>] __blk_run_queue+0x39/0x50
[<ffffffff8d31d566>] blk_run_queue+0x26/0x40
[<ffffffff8d4a7b18>] scsi_run_queue+0x258/0x2f0
[<ffffffff8d4a9770>] scsi_next_command+0x20/0x40
[<ffffffff8d4a98c5>] scsi_end_request+0x135/0x1e0
[<ffffffff8d4a9b38>] scsi_io_completion+0x168/0x6a0
[<ffffffff8d49f01c>] scsi_finish_command+0xdc/0x140
[<ffffffff8d4a9082>] scsi_softirq_done+0x132/0x160
[<ffffffff8d3271a6>] blk_done_softirq+0x96/0xc0
[<ffffffff8d09dba5>] __do_softirq+0xf5/0x280
[<ffffffff8d09dd68>] run_ksoftirqd+0x38/0x50
[<ffffffff8d0c6744>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8d0c6600>] ? lg_double_unlock+0x40/0x40
[<ffffffff8d0bdf21>] kthread+0xd1/0xe0
[<ffffffff8d0bde50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8d7255dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff8d0bde50>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #412189
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01583be
Build id: 3.10.0-1160.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc01c771e
Build id: 3.10.0-1160.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc02762e0
Build id: 3.10.0-1160.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffb12e3980
Build id: 3.10.0-1160.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb12ed0ac
Build id: 3.10.0-1160.el7.x86_64
0x4ac
6
? kmem_cache_free
vmlinux 0xffffffffb1026642
Build id: 3.10.0-1160.el7.x86_64
0x1e2
7
__blk_run_queue
vmlinux 0xffffffffb11534d9
Build id: 3.10.0-1160.el7.x86_64
0x39
8
blk_run_queue
vmlinux 0xffffffffb1153556
Build id: 3.10.0-1160.el7.x86_64
0x26
9
scsi_run_queue
vmlinux 0xffffffffb12eb608
Build id: 3.10.0-1160.el7.x86_64
0x258
10
scsi_next_command
vmlinux 0xffffffffb12ed2c0
Build id: 3.10.0-1160.el7.x86_64
0x20
11
scsi_end_request
vmlinux 0xffffffffb12ed415
Build id: 3.10.0-1160.el7.x86_64
0x135
12
scsi_io_completion
vmlinux 0xffffffffb12ed688
Build id: 3.10.0-1160.el7.x86_64
0x168
13
scsi_finish_command
vmlinux 0xffffffffb12e297c
Build id: 3.10.0-1160.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffffb12ecbd0
Build id: 3.10.0-1160.el7.x86_64
0x130
15
blk_done_softirq
vmlinux 0xffffffffb115d226
Build id: 3.10.0-1160.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffffb0ea4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffffb0ea4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffffb0ece674
Build id: 3.10.0-1160.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffffb0ece530
Build id: 3.10.0-1160.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffffb0ec5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffffb0ec5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffffb1593df7
Build id: 3.10.0-1160.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffffb0ec5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 21s! [ksoftirqd/3:24]
Modules linked in: veth br_netfilter tun devlink overlay(T) bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec iosf_mbi crc32_pclmul ac97_bus ghash_clmulni_intel snd_seq snd_seq_device aesni_intel snd_pcm lrw gf128mul ppdev glue_helper ablk_helper cryptd snd_timer vmw_balloon snd soundcore joydev sg pcspkr i2c_piix4 vmw_vmci parport_pc parport xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common syscopyarea sysfillrect crc32c_intel sysimgblt fb_sys_fops ttm nfit libnvdimm drm serio_raw mptspi e1000 scsi_transport_spi ata_piix mptscsih libata mptbase drm_panel_orientation_quirks
fuse [last unloaded: ip_tables]
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G    B          ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9c85b2fdb180 ti: ffff9c85b2fe8000 task.ti: ffff9c85b2fe8000
RIP: 0010:[<ffffffffc01583be>]  [<ffffffffc01583be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9c85b2febaa0  EFLAGS: 00000246
RAX: ffffaec4006c0000 RBX: ffff9c8489cedfe0 RCX: 0000000000000003
RDX: ffff9c85ac6d8800 RSI: ffff9c85ac454000 RDI: 000000000000000e
RBP: ffff9c85b2febab0 R08: 0000000000000003 R09: 00000000a650ffff
R10: 0000000087986d01 R11: ffff9c8587986bc0 R12: 0000000100000080
R13: ffff9c84832cd380 R14: ffff9c85b2feba14 R15: ffffffffb115bdfb
FS:  0000000000000000(0000) GS:ffff9c85b96c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffc4d9aa5c8 CR3: 000000012c1cc000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc01c771e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02762e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb12e3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb12ed0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffffb1026642>] ? kmem_cache_free+0x1e2/0x200
[<ffffffffb11534d9>] __blk_run_queue+0x39/0x50
[<ffffffffb1153556>] blk_run_queue+0x26/0x40
[<ffffffffb12eb608>] scsi_run_queue+0x258/0x2f0
[<ffffffffb12ed2c0>] scsi_next_command+0x20/0x40
[<ffffffffb12ed415>] scsi_end_request+0x135/0x1e0
[<ffffffffb12ed688>] scsi_io_completion+0x168/0x720
[<ffffffffb12e297c>] scsi_finish_command+0xdc/0x140
[<ffffffffb12ecbd0>] scsi_softirq_done+0x130/0x160
[<ffffffffb115d226>] blk_done_softirq+0x96/0xc0
[<ffffffffb0ea4b95>] __do_softirq+0xf5/0x280
[<ffffffffb0ea4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffb0ece674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb0ece530>] ? lg_double_unlock+0x40/0x40
[<ffffffffb0ec5c21>] kthread+0xd1/0xe0
[<ffffffffb0ec5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffb1593df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb0ec5b50>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 63 16 
Complete report #409859
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01313be
Build id: 3.10.0-1160.62.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc016871e
Build id: 3.10.0-1160.62.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc014c2e0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff90ae4640
Build id: 3.10.0-1160.62.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff90aedd6c
Build id: 3.10.0-1160.62.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffff90953cf9
Build id: 3.10.0-1160.62.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff90953d76
Build id: 3.10.0-1160.62.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff90aec2c8
Build id: 3.10.0-1160.62.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff90aedf80
Build id: 3.10.0-1160.62.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff90aee0d5
Build id: 3.10.0-1160.62.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff90aee348
Build id: 3.10.0-1160.62.1.el7.x86_64
0x168
12
? __netif_receive_skb
vmlinux 0xffffffff90c59b18
Build id: 3.10.0-1160.62.1.el7.x86_64
0x18
13
scsi_finish_command
vmlinux 0xffffffff90ae363c
Build id: 3.10.0-1160.62.1.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffff90aed890
Build id: 3.10.0-1160.62.1.el7.x86_64
0x130
15
blk_done_softirq
vmlinux 0xffffffff9095db16
Build id: 3.10.0-1160.62.1.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffff906a4d85
Build id: 3.10.0-1160.62.1.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffff906a4f48
Build id: 3.10.0-1160.62.1.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff906cea04
Build id: 3.10.0-1160.62.1.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffff906ce8c0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffff906c5fb1
Build id: 3.10.0-1160.62.1.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffff906c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffff90d99df7
Build id: 3.10.0-1160.62.1.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffff906c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ksoftirqd/2:19]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_13402(E) xt_nat veth ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter binfmt_misc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop overlay ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel
lrw ppdev gf128mul vmw_balloon glue_helper ablk_helper cryptd vmw_vmci pcspkr joydev sg i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul libata crct10dif_common nfit libnvdimm crc32c_intel mptspi serio_raw scsi_transport_spi mptscsih vmxnet3 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: P            EL ------------   3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffffa0c175c18000 ti: ffffa0c175c14000 task.ti: ffffa0c175c14000
RIP: 0010:[<ffffffffc01313be>]  [<ffffffffc01313be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffffa0c175c17aa0  EFLAGS: 00000246
RAX: ffffb13a81c40000 RBX: ffffa0c175c17a48 RCX: 000000000000000f
RDX: ffffa0c036603400 RSI: ffffa0c2eb7d1000 RDI: 000000000000000e
RBP: ffffa0c175c17ab0 R08: 0000000000000003 R09: 00000001219d0fff
R10: ffffa0c43fd23c50 R11: ffffd78649a1c800 R12: ffffffff906d7572
R13: ffffa0c175c17a20 R14: 0000000000000002 R15: ffffa0c43fd1acc0
FS:  0000000000000000(0000) GS:ffffa0c43fd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000369f24986000 CR3: 00000003e27e0000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc016871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc014c2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff90ae4640>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff90aedd6c>] scsi_request_fn+0x4ac/0x680
[<ffffffff90953cf9>] __blk_run_queue+0x39/0x50
[<ffffffff90953d76>] blk_run_queue+0x26/0x40
[<ffffffff90aec2c8>] scsi_run_queue+0x258/0x2f0
[<ffffffff90aedf80>] scsi_next_command+0x20/0x40
[<ffffffff90aee0d5>] scsi_end_request+0x135/0x1e0
[<ffffffff90aee348>] scsi_io_completion+0x168/0x720
[<ffffffff90c59b18>] ? __netif_receive_skb+0x18/0x60
[<ffffffff90ae363c>] scsi_finish_command+0xdc/0x140
[<ffffffff90aed890>] scsi_softirq_done+0x130/0x160
[<ffffffff9095db16>] blk_done_softirq+0x96/0xc0
[<ffffffff906a4d85>] __do_softirq+0xf5/0x280
[<ffffffff906a4f48>] run_ksoftirqd+0x38/0x50
[<ffffffff906cea04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff906ce8c0>] ? lg_double_unlock+0x40/0x40
[<ffffffff906c5fb1>] kthread+0xd1/0xe0
[<ffffffff906c5ee0>] ? insert_kthread_work+0x40/0x40
[<ffffffff90d99df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff906c5ee0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 f3 13 
Complete report #406509
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffb398e9f5
Build id: 3.10.0-1160.62.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0317632
Build id: 3.10.0-1160.62.1.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffffb36ecd50
Build id: 3.10.0-1160.62.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffffb36e4640
Build id: 3.10.0-1160.62.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb36edd6c
Build id: 3.10.0-1160.62.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffb3553cf9
Build id: 3.10.0-1160.62.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffb3553d76
Build id: 3.10.0-1160.62.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffb36ec2c8
Build id: 3.10.0-1160.62.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffb36edf80
Build id: 3.10.0-1160.62.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffb36ee0d5
Build id: 3.10.0-1160.62.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffb36ee348
Build id: 3.10.0-1160.62.1.el7.x86_64
0x168
12
scsi_finish_command
vmlinux 0xffffffffb36e363c
Build id: 3.10.0-1160.62.1.el7.x86_64
0xdc
13
scsi_softirq_done
vmlinux 0xffffffffb36ed890
Build id: 3.10.0-1160.62.1.el7.x86_64
0x130
14
blk_done_softirq
vmlinux 0xffffffffb355db16
Build id: 3.10.0-1160.62.1.el7.x86_64
0x96
15
__do_softirq
vmlinux 0xffffffffb32a4d85
Build id: 3.10.0-1160.62.1.el7.x86_64
0xf5
16
run_ksoftirqd
vmlinux 0xffffffffb32a4f48
Build id: 3.10.0-1160.62.1.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffffb32cea04
Build id: 3.10.0-1160.62.1.el7.x86_64
0x144
18
? lg_double_unlock
vmlinux 0xffffffffb32ce8c0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffffb32c5fb1
Build id: 3.10.0-1160.62.1.el7.x86_64
0xd1
20
? insert_kthread_work
vmlinux 0xffffffffb32c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
21
ret_from_fork_nospec_begin
vmlinux 0xffffffffb3999df7
Build id: 3.10.0-1160.62.1.el7.x86_64
0x21
22
? insert_kthread_work
vmlinux 0xffffffffb32c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [ksoftirqd/1:14]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_13109(E) tcp_lp falcon_lsm_pinned_12905(E) nfsv3 nfs_acl nfs lockd grace fscache xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd pcspkr joydev sg parport_pc vmw_vmci parport i2c_piix4 binfmt_misc auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic
drm_kms_helper ahci crct10dif_pclmul crct10dif_common syscopyarea sysfillrect ata_piix libahci sysimgblt fb_sys_fops crc32c_intel ttm serio_raw libata drm vmxnet3 nfit libnvdimm vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: falcon_kal]
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: P            EL ------------   3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff918c38550000 ti: ffff918c38558000 task.ti: ffff918c38558000
RIP: 0010:[<ffffffffb398e9f5>]  [<ffffffffb398e9f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff918c3855bb08  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff918c3855bac8 RCX: ffff918c38139060
RDX: ffffb6c540780000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff918c3855bb08 R08: 0000000000000000 R09: ffff918bb22aea00
R10: ffff918b34815c00 R11: ffff918c33752800 R12: 0000000000000020
R13: ffff918c0ea39e00 R14: ffffffffb36eba09 R15: ffff918c3855ba80
FS:  0000000000000000(0000) GS:ffff918c3fd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6ca63da000 CR3: 000000008b086000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc0317632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffb36ecd50>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffb36e4640>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb36edd6c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb3553cf9>] __blk_run_queue+0x39/0x50
[<ffffffffb3553d76>] blk_run_queue+0x26/0x40
[<ffffffffb36ec2c8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb36edf80>] scsi_next_command+0x20/0x40
[<ffffffffb36ee0d5>] scsi_end_request+0x135/0x1e0
[<ffffffffb36ee348>] scsi_io_completion+0x168/0x720
[<ffffffffb36e363c>] scsi_finish_command+0xdc/0x140
[<ffffffffb36ed890>] scsi_softirq_done+0x130/0x160
[<ffffffffb355db16>] blk_done_softirq+0x96/0xc0
[<ffffffffb32a4d85>] __do_softirq+0xf5/0x280
[<ffffffffb32a4f48>] run_ksoftirqd+0x38/0x50
[<ffffffffb32cea04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb32ce8c0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb32c5fb1>] kthread+0xd1/0xe0
[<ffffffffb32c5ee0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb3999df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb32c5ee0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48 
Complete report #404016
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff8678ea05
Build id: 3.10.0-1160.59.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc01a5632
Build id: 3.10.0-1160.59.1.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff864ecf80
Build id: 3.10.0-1160.59.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff864e4870
Build id: 3.10.0-1160.59.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff864edf9c
Build id: 3.10.0-1160.59.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffff86353f29
Build id: 3.10.0-1160.59.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff86353fa6
Build id: 3.10.0-1160.59.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff864ec4f8
Build id: 3.10.0-1160.59.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff864ee1b0
Build id: 3.10.0-1160.59.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff864ee305
Build id: 3.10.0-1160.59.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff864ee578
Build id: 3.10.0-1160.59.1.el7.x86_64
0x168
12
? switch_mm_irqs_off
vmlinux 0xffffffff8607dfe9
Build id: 3.10.0-1160.59.1.el7.x86_64
0x109
13
? switch_mm_irqs_off
vmlinux 0xffffffff8607dff5
Build id: 3.10.0-1160.59.1.el7.x86_64
0x115
14
scsi_finish_command
vmlinux 0xffffffff864e386c
Build id: 3.10.0-1160.59.1.el7.x86_64
0xdc
15
scsi_softirq_done
vmlinux 0xffffffff864edac0
Build id: 3.10.0-1160.59.1.el7.x86_64
0x130
16
blk_done_softirq
vmlinux 0xffffffff8635dd46
Build id: 3.10.0-1160.59.1.el7.x86_64
0x96
17
__do_softirq
vmlinux 0xffffffff860a4bf5
Build id: 3.10.0-1160.59.1.el7.x86_64
0xf5
18
run_ksoftirqd
vmlinux 0xffffffff860a4db8
Build id: 3.10.0-1160.59.1.el7.x86_64
0x38
19
smpboot_thread_fn
vmlinux 0xffffffff860ce8b4
Build id: 3.10.0-1160.59.1.el7.x86_64
0x144
20
? lg_double_unlock
vmlinux 0xffffffff860ce770
Build id: 3.10.0-1160.59.1.el7.x86_64
0x40
21
kthread
vmlinux 0xffffffff860c5e61
Build id: 3.10.0-1160.59.1.el7.x86_64
0xd1
22
? insert_kthread_work
vmlinux 0xffffffff860c5d90
Build id: 3.10.0-1160.59.1.el7.x86_64
0x40
23
ret_from_fork_nospec_begin
vmlinux 0xffffffff86799df7
Build id: 3.10.0-1160.59.1.el7.x86_64
0x21
24
? insert_kthread_work
vmlinux 0xffffffff860c5d90
Build id: 3.10.0-1160.59.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink ipmi_devintf ipmi_msghandler vmw_vsock_vmci_transport vsock sunrpc iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon sg joydev pcspkr parport_pc parport i2c_piix4 vmw_vmci binfmt_misc ip_tables xfs libcrc32c ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common
e1000e drm crc32c_intel serio_raw ahci ptp pps_core libahci vmw_pvscsi ata_piix drm_panel_orientation_quirks nfit libata libnvdimm dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/28/2017
task: ffff940b76315280 ti: ffff940b7634c000 task.ti: ffff940b7634c000
RIP: 0010:[<ffffffff8678ea05>]  [<ffffffff8678ea05>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff940b7634fb08  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff940b7634fac8 RCX: ffff940ba6bc9060
RDX: ffffb37fc1b80000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff940b7634fb08 R08: 0000000000000000 R09: ffff940a90995c00
R10: ffff940ba6582000 R11: ffff940ba660f000 R12: 0000000000000020
R13: ffff940d301d5500 R14: ffffffff864ebc39 R15: ffff940b7634fa80
FS:  0000000000000000(0000) GS:ffff940e3fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000006dee20 CR3: 00000003f2c94000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc01a5632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff864ecf80>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff864e4870>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff864edf9c>] scsi_request_fn+0x4ac/0x680
[<ffffffff86353f29>] __blk_run_queue+0x39/0x50
[<ffffffff86353fa6>] blk_run_queue+0x26/0x40
[<ffffffff864ec4f8>] scsi_run_queue+0x258/0x2f0
[<ffffffff864ee1b0>] scsi_next_command+0x20/0x40
[<ffffffff864ee305>] scsi_end_request+0x135/0x1e0
[<ffffffff864ee578>] scsi_io_completion+0x168/0x720
[<ffffffff8607dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffff8607dff5>] ? switch_mm_irqs_off+0x115/0x290
[<ffffffff864e386c>] scsi_finish_command+0xdc/0x140
[<ffffffff864edac0>] scsi_softirq_done+0x130/0x160
[<ffffffff8635dd46>] blk_done_softirq+0x96/0xc0
[<ffffffff860a4bf5>] __do_softirq+0xf5/0x280
[<ffffffff860a4db8>] run_ksoftirqd+0x38/0x50
[<ffffffff860ce8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff860ce770>] ? lg_double_unlock+0x40/0x40
[<ffffffff860c5e61>] kthread+0xd1/0xe0
[<ffffffff860c5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffff86799df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff860c5d90>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #393313
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff816adce5
Build id: 3.10.0-693.17.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc001b632
Build id: 3.10.0-693.17.1.el7.x86_64
0x442
3
? scsi_test_unit_ready
vmlinux 0xffffffff8146f5b0
Build id: 3.10.0-693.17.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff8146782a
Build id: 3.10.0-693.17.1.el7.x86_64
0xaa
5
scsi_request_fn
vmlinux 0xffffffff814705ef
Build id: 3.10.0-693.17.1.el7.x86_64
0x4df
6
__blk_run_queue
vmlinux 0xffffffff812f96a3
Build id: 3.10.0-693.17.1.el7.x86_64
0x33
7
blk_run_queue
vmlinux 0xffffffff812f9716
Build id: 3.10.0-693.17.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff8146eb08
Build id: 3.10.0-693.17.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff814707d0
Build id: 3.10.0-693.17.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff81470925
Build id: 3.10.0-693.17.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff81470d07
Build id: 3.10.0-693.17.1.el7.x86_64
0x2d7
12
scsi_finish_command
vmlinux 0xffffffff81466845
Build id: 3.10.0-693.17.1.el7.x86_64
0xd5
13
scsi_softirq_done
vmlinux 0xffffffff814700e2
Build id: 3.10.0-693.17.1.el7.x86_64
0x132
14
blk_done_softirq
vmlinux 0xffffffff813029e0
Build id: 3.10.0-693.17.1.el7.x86_64
0x90
15
__do_softirq
vmlinux 0xffffffff8109275f
Build id: 3.10.0-693.17.1.el7.x86_64
0xef
16
run_ksoftirqd
vmlinux 0xffffffff81092928
Build id: 3.10.0-693.17.1.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffff810bae1f
Build id: 3.10.0-693.17.1.el7.x86_64
0x12f
18
? lg_double_unlock
vmlinux 0xffffffff810bacf0
Build id: 3.10.0-693.17.1.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffff810b270f
Build id: 3.10.0-693.17.1.el7.x86_64
0xcf
20
? insert_kthread_work
vmlinux 0xffffffff810b2640
Build id: 3.10.0-693.17.1.el7.x86_64
0x40
21
ret_from_fork
vmlinux 0xffffffff816b8798
Build id: 3.10.0-693.17.1.el7.x86_64
0x58
22
? insert_kthread_work
vmlinux 0xffffffff810b2640
Build id: 3.10.0-693.17.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#12 stuck for 54s! [ksoftirqd/12:68]
Modules linked in: vfat fat isofs xfs libcrc32c binfmt_misc bmhook(OE) tmhook(OE) dsa_filter(POE) dsa_filter_hook(OE) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod btrfs sb_edac edac_core iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd raid6_pq xor joydev pcspkr sg nfit parport_pc parport libnvdimm vmw_vmci shpchp i2c_piix4 ip_tables ext4 mbcache jbd2 sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ahci ata_piix libahci libata serio_raw vmxnet3 i2c_core vmw_pvscsi floppy
CPU: 12 PID: 68 Comm: ksoftirqd/12 Tainted: P           OEL ------------   3.10.0-693.17.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff880176a89fa0 ti: ffff880176a98000 task.ti: ffff880176a98000
RIP: 0010:[<ffffffff816adce5>]  [<ffffffff816adce5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff880176a9bb08  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff880176a9bac8 RCX: 00000005a248efff
RDX: ffffc90003190000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff880176a9bb08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8805a15d6d80 R11: ffff8805a4cf5800 R12: 0000000000000020
R13: ffff88019a524d80 R14: ffffffff8146e1d9 R15: ffff880176a9ba80
FS:  0000000000000000(0000) GS:ffff8805a7900000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f83bffa3000 CR3: 00000000019fa000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc001b632>] pvscsi_queue+0x442/0x7e1 [vmw_pvscsi]
[<ffffffff8146f5b0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8146782a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff814705ef>] scsi_request_fn+0x4df/0x680
[<ffffffff812f96a3>] __blk_run_queue+0x33/0x40
[<ffffffff812f9716>] blk_run_queue+0x26/0x40
[<ffffffff8146eb08>] scsi_run_queue+0x258/0x2f0
[<ffffffff814707d0>] scsi_next_command+0x20/0x40
[<ffffffff81470925>] scsi_end_request+0x135/0x1e0
[<ffffffff81470d07>] scsi_io_completion+0x2d7/0x6a0
[<ffffffff81466845>] scsi_finish_command+0xd5/0x130
[<ffffffff814700e2>] scsi_softirq_done+0x132/0x160
[<ffffffff813029e0>] blk_done_softirq+0x90/0xc0
[<ffffffff8109275f>] __do_softirq+0xef/0x280
[<ffffffff81092928>] run_ksoftirqd+0x38/0x50
[<ffffffff810bae1f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810bacf0>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b270f>] kthread+0xcf/0xe0
[<ffffffff810b2640>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b8798>] ret_from_fork+0x58/0x90
[<ffffffff810b2640>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #388785
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff9e3893e5
Build id: 3.10.0-1160.11.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0080632
Build id: 3.10.0-1160.11.1.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff9e0ec2a0
Build id: 3.10.0-1160.11.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff9e0e3b90
Build id: 3.10.0-1160.11.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff9e0ed2bc
Build id: 3.10.0-1160.11.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffff9df536d9
Build id: 3.10.0-1160.11.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff9df53756
Build id: 3.10.0-1160.11.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff9e0eb818
Build id: 3.10.0-1160.11.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff9e0ed4d0
Build id: 3.10.0-1160.11.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff9e0ed625
Build id: 3.10.0-1160.11.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff9e0ed898
Build id: 3.10.0-1160.11.1.el7.x86_64
0x168
12
? __netif_receive_skb
vmlinux 0xffffffff9e254b28
Build id: 3.10.0-1160.11.1.el7.x86_64
0x18
13
scsi_finish_command
vmlinux 0xffffffff9e0e2b8c
Build id: 3.10.0-1160.11.1.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffff9e0ecde0
Build id: 3.10.0-1160.11.1.el7.x86_64
0x130
15
blk_done_softirq
vmlinux 0xffffffff9df5d426
Build id: 3.10.0-1160.11.1.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffff9dca4b95
Build id: 3.10.0-1160.11.1.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffff9dca4d58
Build id: 3.10.0-1160.11.1.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff9dcce8c4
Build id: 3.10.0-1160.11.1.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffff9dcce780
Build id: 3.10.0-1160.11.1.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffff9dcc5e71
Build id: 3.10.0-1160.11.1.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffff9dcc5da0
Build id: 3.10.0-1160.11.1.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffff9e393ddd
Build id: 3.10.0-1160.11.1.el7.x86_64
0x7
23
? insert_kthread_work
vmlinux 0xffffffff9dcc5da0
Build id: 3.10.0-1160.11.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [ksoftirqd/4:29]
Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache xt_set nf_tables ip6t_MASQUERADE nf_nat_masquerade_ipv6 xt_comment xt_mark ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack_netlink xt_addrtype xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables veth vxlan ip6_udp_tunnel udp_tunnel ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set_bitmap_port
dummy br_netfilter tun devlink overlay(T) binfmt_misc bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg joydev pcspkr vmw_vmci vmw_balloon parport_pc i2c_piix4 parport xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci nfit libnvdimm sd_mod libahci ata_piix crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common libata crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: ip_tables]
CPU: 4 PID: 29 Comm: ksoftirqd/4 Kdump: loaded Tainted: G        W    L ------------ T 3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff92f9fe758000 ti: ffff92f9fe760000 task.ti: ffff92f9fe760000
RIP: 0010:[<ffffffff9e3893e5>]  [<ffffffff9e3893e5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff92f9fe763b08  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff92f9fe763ac8 RCX: ffffffff9e834f40
RDX: ffffb56e00ea0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff92f9fe763b08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff92f9efbafc00 R11: ffff92fab469d800 R12: 0000000000000020
R13: ffff92fa3eeab080 R14: ffffffff9e0eaf59 R15: ffff92f9fe763a80
FS:  0000000000000000(0000) GS:ffff92fabfd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000562ee072250f CR3: 0000000042410000 CR4: 00000000007607e0
PKRU: 00000000
Call Trace:
[<ffffffffc0080632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff9e0ec2a0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff9e0e3b90>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9e0ed2bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff9df536d9>] __blk_run_queue+0x39/0x50
[<ffffffff9df53756>] blk_run_queue+0x26/0x40
[<ffffffff9e0eb818>] scsi_run_queue+0x258/0x2f0
[<ffffffff9e0ed4d0>] scsi_next_command+0x20/0x40
[<ffffffff9e0ed625>] scsi_end_request+0x135/0x1e0
[<ffffffff9e0ed898>] scsi_io_completion+0x168/0x720
[<ffffffff9e254b28>] ? __netif_receive_skb+0x18/0x60
[<ffffffff9e0e2b8c>] scsi_finish_command+0xdc/0x140
[<ffffffff9e0ecde0>] scsi_softirq_done+0x130/0x160
[<ffffffff9df5d426>] blk_done_softirq+0x96/0xc0
[<ffffffff9dca4b95>] __do_softirq+0xf5/0x280
[<ffffffff9dca4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff9dcce8c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9dcce780>] ? lg_double_unlock+0x40/0x40
[<ffffffff9dcc5e71>] kthread+0xd1/0xe0
[<ffffffff9dcc5da0>] ? insert_kthread_work+0x40/0x40
[<ffffffff9e393ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff9dcc5da0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #353790
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc04453be
Build id: 3.10.0-1160.53.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc04bb71e
Build id: 3.10.0-1160.53.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc04282e0
Build id: 3.10.0-1160.53.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff974e4860
Build id: 3.10.0-1160.53.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff974edf8c
Build id: 3.10.0-1160.53.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffff97353f39
Build id: 3.10.0-1160.53.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff97353fb6
Build id: 3.10.0-1160.53.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff974ec4e8
Build id: 3.10.0-1160.53.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff974ee1a0
Build id: 3.10.0-1160.53.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff974ee2f5
Build id: 3.10.0-1160.53.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff974ee568
Build id: 3.10.0-1160.53.1.el7.x86_64
0x168
12
? __enqueue_entity
vmlinux 0xffffffff970dfbd8
Build id: 3.10.0-1160.53.1.el7.x86_64
0x78
13
scsi_finish_command
vmlinux 0xffffffff974e385c
Build id: 3.10.0-1160.53.1.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffff974edab0
Build id: 3.10.0-1160.53.1.el7.x86_64
0x130
15
blk_done_softirq
vmlinux 0xffffffff9735dd36
Build id: 3.10.0-1160.53.1.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffff970a4bf5
Build id: 3.10.0-1160.53.1.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffff970a4db8
Build id: 3.10.0-1160.53.1.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff970ce8b4
Build id: 3.10.0-1160.53.1.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffff970ce770
Build id: 3.10.0-1160.53.1.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffff970c5e61
Build id: 3.10.0-1160.53.1.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffff970c5d90
Build id: 3.10.0-1160.53.1.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffff97795df7
Build id: 3.10.0-1160.53.1.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffff970c5d90
Build id: 3.10.0-1160.53.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#30 stuck for 22s! [ksoftirqd/30:163]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon pcspkr sg joydev parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi scsi_transport_spi mptscsih libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks
floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 30 PID: 163 Comm: ksoftirqd/30 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.53.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff99e01e79e300 ti: ffff99e01e7d0000 task.ti: ffff99e01e7d0000
RIP: 0010:[<ffffffffc04453be>]  [<ffffffffc04453be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff99e01e7d3aa0  EFLAGS: 00000246
RAX: ffffa7c841c40000 RBX: ffff99e03fde9100 RCX: 0000000000000009
RDX: ffff99df9c368000 RSI: ffff99def9e50000 RDI: 000000000000000e
RBP: ffff99e01e7d3ab0 R08: 0000000000000002 R09: 0000000016739fff
R10: ffff99e03b753c00 R11: ffff99e03b4c1c00 R12: 0000000100000001
R13: ffff99ddcbbc4000 R14: ffff99e01e7d3a14 R15: ffffffff9735c8fb
FS:  0000000000000000(0000) GS:ffff99e03fd80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd1ba14e7a0 CR3: 00000001e7610000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc04bb71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04282e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff974e4860>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff974edf8c>] scsi_request_fn+0x4ac/0x680
[<ffffffff97353f39>] __blk_run_queue+0x39/0x50
[<ffffffff97353fb6>] blk_run_queue+0x26/0x40
[<ffffffff974ec4e8>] scsi_run_queue+0x258/0x2f0
[<ffffffff974ee1a0>] scsi_next_command+0x20/0x40
[<ffffffff974ee2f5>] scsi_end_request+0x135/0x1e0
[<ffffffff974ee568>] scsi_io_completion+0x168/0x720
[<ffffffff970dfbd8>] ? __enqueue_entity+0x78/0x80
[<ffffffff974e385c>] scsi_finish_command+0xdc/0x140
[<ffffffff974edab0>] scsi_softirq_done+0x130/0x160
[<ffffffff9735dd36>] blk_done_softirq+0x96/0xc0
[<ffffffff970a4bf5>] __do_softirq+0xf5/0x280
[<ffffffff970a4db8>] run_ksoftirqd+0x38/0x50
[<ffffffff970ce8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff970ce770>] ? lg_double_unlock+0x40/0x40
[<ffffffff970c5e61>] kthread+0xd1/0xe0
[<ffffffff970c5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffff97795df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff970c5d90>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 33 45 
Complete report #323426
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc036d3be
Build id: 3.10.0-1160.49.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc038f71e
Build id: 3.10.0-1160.49.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03df2e0
Build id: 3.10.0-1160.49.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffb22e4860
Build id: 3.10.0-1160.49.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb22edf8c
Build id: 3.10.0-1160.49.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffb2153f39
Build id: 3.10.0-1160.49.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffb2153fb6
Build id: 3.10.0-1160.49.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffb22ec4e8
Build id: 3.10.0-1160.49.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffb22ee1a0
Build id: 3.10.0-1160.49.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffb22ee2f5
Build id: 3.10.0-1160.49.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffb22ee568
Build id: 3.10.0-1160.49.1.el7.x86_64
0x168
12
? switch_mm_irqs_off
vmlinux 0xffffffffb1e7dfe9
Build id: 3.10.0-1160.49.1.el7.x86_64
0x109
13
? switch_mm_irqs_off
vmlinux 0xffffffffb1e7dff5
Build id: 3.10.0-1160.49.1.el7.x86_64
0x115
14
scsi_finish_command
vmlinux 0xffffffffb22e385c
Build id: 3.10.0-1160.49.1.el7.x86_64
0xdc
15
scsi_softirq_done
vmlinux 0xffffffffb22edab0
Build id: 3.10.0-1160.49.1.el7.x86_64
0x130
16
blk_done_softirq
vmlinux 0xffffffffb215dd36
Build id: 3.10.0-1160.49.1.el7.x86_64
0x96
17
__do_softirq
vmlinux 0xffffffffb1ea4bf5
Build id: 3.10.0-1160.49.1.el7.x86_64
0xf5
18
run_ksoftirqd
vmlinux 0xffffffffb1ea4db8
Build id: 3.10.0-1160.49.1.el7.x86_64
0x38
19
smpboot_thread_fn
vmlinux 0xffffffffb1ece8b4
Build id: 3.10.0-1160.49.1.el7.x86_64
0x144
20
? lg_double_unlock
vmlinux 0xffffffffb1ece770
Build id: 3.10.0-1160.49.1.el7.x86_64
0x40
21
kthread
vmlinux 0xffffffffb1ec5e61
Build id: 3.10.0-1160.49.1.el7.x86_64
0xd1
22
? insert_kthread_work
vmlinux 0xffffffffb1ec5d90
Build id: 3.10.0-1160.49.1.el7.x86_64
0x40
23
ret_from_fork_nospec_begin
vmlinux 0xffffffffb2595de4
Build id: 3.10.0-1160.49.1.el7.x86_64
0xe
24
? insert_kthread_work
vmlinux 0xffffffffb1ec5d90
Build id: 3.10.0-1160.49.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 592s! [ksoftirqd/3:24]
Modules linked in: tcp_lp rfcomm xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel ppdev vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus btusb joydev btrtl snd_seq btbcm btintel pcspkr snd_seq_device
bluetooth snd_pcm rfkill snd_timer snd soundcore sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel drm_kms_helper syscopyarea ata_piix sysfillrect sysimgblt serio_raw fb_sys_fops ttm e1000 drm nfit libata mptspi libnvdimm scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff98b1f8aeb180 ti: ffff98b1f8b0c000 task.ti: ffff98b1f8b0c000
RIP: 0010:[<ffffffffc036d3be>]  [<ffffffffc036d3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff98b1f8b0faa0  EFLAGS: 00000246
RAX: ffffa97c80380000 RBX: ffffffffb1ec6f7b RCX: 0000000000000000
RDX: ffff98b1b541a800 RSI: ffff98b1f83b7000 RDI: 000000000000000e
RBP: ffff98b1f8b0fab0 R08: 0000000000000003 R09: 000000006e5fbfff
R10: 00000000f5964501 R11: ffff98b1f5967640 R12: ffffffffb1edae02
R13: ffff98b1f8b0fa18 R14: ffff98b1fffb1530 R15: ffff98b1f8b0fac0
FS:  0000000000000000(0000) GS:ffff98b1fdec0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007faa15ded2f0 CR3: 0000000025556000 CR4: 00000000003407e0
Call Trace:
[<ffffffffc038f71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03df2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb22e4860>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb22edf8c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb2153f39>] __blk_run_queue+0x39/0x50
[<ffffffffb2153fb6>] blk_run_queue+0x26/0x40
[<ffffffffb22ec4e8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb22ee1a0>] scsi_next_command+0x20/0x40
[<ffffffffb22ee2f5>] scsi_end_request+0x135/0x1e0
[<ffffffffb22ee568>] scsi_io_completion+0x168/0x720
[<ffffffffb1e7dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffffb1e7dff5>] ? switch_mm_irqs_off+0x115/0x290
[<ffffffffb22e385c>] scsi_finish_command+0xdc/0x140
[<ffffffffb22edab0>] scsi_softirq_done+0x130/0x160
[<ffffffffb215dd36>] blk_done_softirq+0x96/0xc0
[<ffffffffb1ea4bf5>] __do_softirq+0xf5/0x280
[<ffffffffb1ea4db8>] run_ksoftirqd+0x38/0x50
[<ffffffffb1ece8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb1ece770>] ? lg_double_unlock+0x40/0x40
[<ffffffffb1ec5e61>] kthread+0xd1/0xe0
[<ffffffffb1ec5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffffb2595de4>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffffb1ec5d90>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 37 
Complete report #322757
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc036d3be
Build id: 3.10.0-1160.49.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc038f71e
Build id: 3.10.0-1160.49.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03df2e0
Build id: 3.10.0-1160.49.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffb22e4860
Build id: 3.10.0-1160.49.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb22edf8c
Build id: 3.10.0-1160.49.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffb2153f39
Build id: 3.10.0-1160.49.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffb2153fb6
Build id: 3.10.0-1160.49.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffb22ec4e8
Build id: 3.10.0-1160.49.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffb22ee1a0
Build id: 3.10.0-1160.49.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffb22ee2f5
Build id: 3.10.0-1160.49.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffb22ee568
Build id: 3.10.0-1160.49.1.el7.x86_64
0x168
12
? switch_mm_irqs_off
vmlinux 0xffffffffb1e7dfe9
Build id: 3.10.0-1160.49.1.el7.x86_64
0x109
13
scsi_finish_command
vmlinux 0xffffffffb22e385c
Build id: 3.10.0-1160.49.1.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffffb22edab0
Build id: 3.10.0-1160.49.1.el7.x86_64
0x130
15
blk_done_softirq
vmlinux 0xffffffffb215dd36
Build id: 3.10.0-1160.49.1.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffffb1ea4bf5
Build id: 3.10.0-1160.49.1.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffffb1ea4db8
Build id: 3.10.0-1160.49.1.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffffb1ece8b4
Build id: 3.10.0-1160.49.1.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffffb1ece770
Build id: 3.10.0-1160.49.1.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffffb1ec5e61
Build id: 3.10.0-1160.49.1.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffffb1ec5d90
Build id: 3.10.0-1160.49.1.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffffb2595de4
Build id: 3.10.0-1160.49.1.el7.x86_64
0xe
23
? insert_kthread_work
vmlinux 0xffffffffb1ec5d90
Build id: 3.10.0-1160.49.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 364s! [ksoftirqd/3:24]
Modules linked in: tcp_lp rfcomm xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel ppdev vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus btusb joydev btrtl snd_seq btbcm btintel pcspkr snd_seq_device
bluetooth snd_pcm rfkill snd_timer snd soundcore sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel drm_kms_helper syscopyarea ata_piix sysfillrect sysimgblt serio_raw fb_sys_fops ttm e1000 drm nfit libata mptspi libnvdimm scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff98b1f8aeb180 ti: ffff98b1f8b0c000 task.ti: ffff98b1f8b0c000
RIP: 0010:[<ffffffffc036d3be>]  [<ffffffffc036d3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff98b1f8b0faa0  EFLAGS: 00000246
RAX: ffffa97c80380000 RBX: ffff98b181eee740 RCX: 0000000000000017
RDX: ffff98b1b541a800 RSI: ffff98b1f83b7000 RDI: 000000000000000e
RBP: ffff98b1f8b0fab0 R08: 0000000000000003 R09: 000000000bdbdfff
R10: ffff98b1fdee3a30 R11: ffffd09d4019ad00 R12: 000000010000001b
R13: ffff98b18639e400 R14: ffff98b1f8b0fa14 R15: ffffffffb215c8fb
FS:  0000000000000000(0000) GS:ffff98b1fdec0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f144d9ff2e0 CR3: 0000000035ff2000 CR4: 00000000003407e0
Call Trace:
[<ffffffffc038f71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03df2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb22e4860>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb22edf8c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb2153f39>] __blk_run_queue+0x39/0x50
[<ffffffffb2153fb6>] blk_run_queue+0x26/0x40
[<ffffffffb22ec4e8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb22ee1a0>] scsi_next_command+0x20/0x40
[<ffffffffb22ee2f5>] scsi_end_request+0x135/0x1e0
[<ffffffffb22ee568>] scsi_io_completion+0x168/0x720
[<ffffffffb1e7dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffffb22e385c>] scsi_finish_command+0xdc/0x140
[<ffffffffb22edab0>] scsi_softirq_done+0x130/0x160
[<ffffffffb215dd36>] blk_done_softirq+0x96/0xc0
[<ffffffffb1ea4bf5>] __do_softirq+0xf5/0x280
[<ffffffffb1ea4db8>] run_ksoftirqd+0x38/0x50
[<ffffffffb1ece8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb1ece770>] ? lg_double_unlock+0x40/0x40
[<ffffffffb1ec5e61>] kthread+0xd1/0xe0
[<ffffffffb1ec5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffffb2595de4>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffffb1ec5d90>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 37 
Complete report #322609
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff9f51b445
Build id: 3.10.0-862.14.4.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0083622
Build id: 3.10.0-862.14.4.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff9f2a8630
Build id: 3.10.0-862.14.4.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff9f2a0140
Build id: 3.10.0-862.14.4.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff9f2a96ac
Build id: 3.10.0-862.14.4.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffff9f11d5e9
Build id: 3.10.0-862.14.4.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff9f11d666
Build id: 3.10.0-862.14.4.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff9f2a7c48
Build id: 3.10.0-862.14.4.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff9f2a98a0
Build id: 3.10.0-862.14.4.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff9f2a99f5
Build id: 3.10.0-862.14.4.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff9f2a9c68
Build id: 3.10.0-862.14.4.el7.x86_64
0x168
12
scsi_finish_command
vmlinux 0xffffffff9f29f14c
Build id: 3.10.0-862.14.4.el7.x86_64
0xdc
13
scsi_softirq_done
vmlinux 0xffffffff9f2a91b2
Build id: 3.10.0-862.14.4.el7.x86_64
0x132
14
blk_done_softirq
vmlinux 0xffffffff9f1272a6
Build id: 3.10.0-862.14.4.el7.x86_64
0x96
15
__do_softirq
vmlinux 0xffffffff9ee9dba5
Build id: 3.10.0-862.14.4.el7.x86_64
0xf5
16
run_ksoftirqd
vmlinux 0xffffffff9ee9dd68
Build id: 3.10.0-862.14.4.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffff9eec6774
Build id: 3.10.0-862.14.4.el7.x86_64
0x144
18
? lg_double_unlock
vmlinux 0xffffffff9eec6630
Build id: 3.10.0-862.14.4.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffff9eebdf21
Build id: 3.10.0-862.14.4.el7.x86_64
0xd1
20
? insert_kthread_work
vmlinux 0xffffffff9eebde50
Build id: 3.10.0-862.14.4.el7.x86_64
0x40
21
ret_from_fork_nospec_begin
vmlinux 0xffffffff9f5255f7
Build id: 3.10.0-862.14.4.el7.x86_64
0x21
22
? insert_kthread_work
vmlinux 0xffffffff9eebde50
Build id: 3.10.0-862.14.4.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 26s! [ksoftirqd/0:3]
Modules linked in: talpa_vfshook(OE) talpa_pedconnector(OE) talpa_pedevice(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) nf_nat_ftp xt_REDIRECT nf_nat_redirect nf_conntrack_ftp ip6table_nat nf_nat_ipv6 iptable_raw xt_connlimit iptable_mangle iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat binfmt_misc tcp_diag inet_diag ip6t_REJECT nf_reject_ipv6 nf_log_ipv6 ip6table_filter nf_log_ipv4 nf_log_common xt_conntrack xt_LOG xt_limit xt_multiport ip6table_mangle ip6table_raw nf_conntrack_ipv6 nf_defrag_ipv6 nf_conntrack ip6_tables ipt_REJECT nf_reject_ipv4 iptable_filter sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr joydev vmw_balloon sg parport_pc parport vmw_vmci
shpchp i2c_piix4 tcp_htcp ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm crct10dif_common crc32c_intel e1000 drm serio_raw ata_piix libata vmw_pvscsi i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: talpa_syscallhook]
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G        W  OEL ------------   3.10.0-862.14.4.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff949afa2b9fa0 ti: ffff949afa334000 task.ti: ffff949afa334000
RIP: 0010:[<ffffffff9f51b445>]  [<ffffffff9f51b445>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0000:ffff949afa337b08  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff949afa337ac8 RCX: ffffffff9fa30880
RDX: ffffaa0dc1b80000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff949afa337b08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff9499b66488d8 R11: ffff949d4638e800 R12: 0000000000000020
R13: ffff949ac5579800 R14: ffffffff9f2a7389 R15: ffff949afa337a80
FS:  0000000000000000(0000) GS:ffff949dbfc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff36a9f6000 CR3: 00000003c640e000 CR4: 00000000000407f0
Call Trace:
[<ffffffffc0083622>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff9f2a8630>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff9f2a0140>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9f2a96ac>] scsi_request_fn+0x4cc/0x680
[<ffffffff9f11d5e9>] __blk_run_queue+0x39/0x50
[<ffffffff9f11d666>] blk_run_queue+0x26/0x40
[<ffffffff9f2a7c48>] scsi_run_queue+0x258/0x2f0
[<ffffffff9f2a98a0>] scsi_next_command+0x20/0x40
[<ffffffff9f2a99f5>] scsi_end_request+0x135/0x1e0
[<ffffffff9f2a9c68>] scsi_io_completion+0x168/0x6a0
[<ffffffff9f29f14c>] scsi_finish_command+0xdc/0x140
[<ffffffff9f2a91b2>] scsi_softirq_done+0x132/0x160
[<ffffffff9f1272a6>] blk_done_softirq+0x96/0xc0
[<ffffffff9ee9dba5>] __do_softirq+0xf5/0x280
[<ffffffff9ee9dd68>] run_ksoftirqd+0x38/0x50
[<ffffffff9eec6774>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9eec6630>] ? lg_double_unlock+0x40/0x40
[<ffffffff9eebdf21>] kthread+0xd1/0xe0
[<ffffffff9eebde50>] ? insert_kthread_work+0x40/0x40
[<ffffffff9f5255f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9eebde50>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #257877
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03283be
Build id: 3.10.0-1160.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc039e71e
Build id: 3.10.0-1160.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc04072e0
Build id: 3.10.0-1160.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8aee3980
Build id: 3.10.0-1160.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff8aeed0ac
Build id: 3.10.0-1160.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffff8ad534d9
Build id: 3.10.0-1160.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff8ad53556
Build id: 3.10.0-1160.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff8aeeb608
Build id: 3.10.0-1160.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff8aeed2c0
Build id: 3.10.0-1160.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff8aeed415
Build id: 3.10.0-1160.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff8aeed688
Build id: 3.10.0-1160.el7.x86_64
0x168
12
? put_prev_entity
vmlinux 0xffffffff8aae5f1b
Build id: 3.10.0-1160.el7.x86_64
0x8b
13
scsi_finish_command
vmlinux 0xffffffff8aee297c
Build id: 3.10.0-1160.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffff8aeecbd0
Build id: 3.10.0-1160.el7.x86_64
0x130
15
blk_done_softirq
vmlinux 0xffffffff8ad5d226
Build id: 3.10.0-1160.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffff8aaa4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffff8aaa4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff8aace674
Build id: 3.10.0-1160.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffff8aace530
Build id: 3.10.0-1160.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffff8aac5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffff8aac5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffff8b193df7
Build id: 3.10.0-1160.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffff8aac5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 24s! [ksoftirqd/3:24]
Modules linked in: veth nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul snd_ens1371 ghash_clmulni_intel snd_rawmidi ppdev snd_ac97_codec ac97_bus snd_seq vmw_balloon aesni_intel lrw snd_seq_device
gf128mul glue_helper ablk_helper cryptd snd_pcm joydev pcspkr snd_timer snd sg i2c_piix4 soundcore vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit crct10dif_pclmul mptspi crct10dif_common crc32c_intel scsi_transport_spi ata_piix drm libnvdimm serio_raw e1000 mptscsih libata mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G        W    L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff8b28397d3180 ti: ffff8b28397e4000 task.ti: ffff8b28397e4000
RIP: 0010:[<ffffffffc03283be>]  [<ffffffffc03283be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8b28397e7aa0  EFLAGS: 00000246
RAX: ffffaeb040440000 RBX: ffff8b27c73e6440 RCX: 0000000000000001
RDX: ffff8b27f4c02800 RSI: ffff8b2836fff000 RDI: 000000000000000e
RBP: ffff8b28397e7ab0 R08: 0000000000000003 R09: 000000006e7f3fff
R10: 00000000c60bc701 R11: ffff8b27c60bc000 R12: 0000000100000023
R13: ffff8b27c770d200 R14: ffff8b28397e7a14 R15: ffffffff8ad5bdfb
FS:  0000000000000000(0000) GS:ffff8b283b6c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f987832f060 CR3: 000000003eb8a000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc039e71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04072e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8aee3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8aeed0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffff8ad534d9>] __blk_run_queue+0x39/0x50
[<ffffffff8ad53556>] blk_run_queue+0x26/0x40
[<ffffffff8aeeb608>] scsi_run_queue+0x258/0x2f0
[<ffffffff8aeed2c0>] scsi_next_command+0x20/0x40
[<ffffffff8aeed415>] scsi_end_request+0x135/0x1e0
[<ffffffff8aeed688>] scsi_io_completion+0x168/0x720
[<ffffffff8aae5f1b>] ? put_prev_entity+0x8b/0x400
[<ffffffff8aee297c>] scsi_finish_command+0xdc/0x140
[<ffffffff8aeecbd0>] scsi_softirq_done+0x130/0x160
[<ffffffff8ad5d226>] blk_done_softirq+0x96/0xc0
[<ffffffff8aaa4b95>] __do_softirq+0xf5/0x280
[<ffffffff8aaa4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff8aace674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8aace530>] ? lg_double_unlock+0x40/0x40
[<ffffffff8aac5c21>] kthread+0xd1/0xe0
[<ffffffff8aac5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8b193df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8aac5b50>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 63 33 
Complete report #241671
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc00503be
Build id: 3.10.0-693.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc009e729
Build id: 3.10.0-693.el7.x86_64
0x249
3
mptspi_qcmd
mptspi 0xffffffffc006b2b0
Build id: 3.10.0-693.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8146540a
Build id: 3.10.0-693.el7.x86_64
0xaa
5
scsi_request_fn
vmlinux 0xffffffff8146e1cf
Build id: 3.10.0-693.el7.x86_64
0x4df
6
? __wake_up_common
vmlinux 0xffffffff810ba588
Build id: 3.10.0-693.el7.x86_64
0x58
7
__blk_run_queue
vmlinux 0xffffffff812f71d3
Build id: 3.10.0-693.el7.x86_64
0x33
8
blk_run_queue
vmlinux 0xffffffff812f7246
Build id: 3.10.0-693.el7.x86_64
0x26
9
scsi_run_queue
vmlinux 0xffffffff8146c6e8
Build id: 3.10.0-693.el7.x86_64
0x258
10
scsi_next_command
vmlinux 0xffffffff8146e3b0
Build id: 3.10.0-693.el7.x86_64
0x20
11
scsi_end_request
vmlinux 0xffffffff8146e505
Build id: 3.10.0-693.el7.x86_64
0x135
12
scsi_io_completion
vmlinux 0xffffffff8146e778
Build id: 3.10.0-693.el7.x86_64
0x168
13
scsi_finish_command
vmlinux 0xffffffff81464425
Build id: 3.10.0-693.el7.x86_64
0xd5
14
scsi_softirq_done
vmlinux 0xffffffff8146dcc2
Build id: 3.10.0-693.el7.x86_64
0x132
15
blk_done_softirq
vmlinux 0xffffffff81300510
Build id: 3.10.0-693.el7.x86_64
0x90
16
__do_softirq
vmlinux 0xffffffff81090b3f
Build id: 3.10.0-693.el7.x86_64
0xef
17
run_ksoftirqd
vmlinux 0xffffffff81090d08
Build id: 3.10.0-693.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff810b909f
Build id: 3.10.0-693.el7.x86_64
0x12f
19
? lg_double_unlock
vmlinux 0xffffffff810b8f70
Build id: 3.10.0-693.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
0xcf
21
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
22
ret_from_fork
vmlinux 0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
0x58
23
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [ksoftirqd/7:43]
Modules linked in: tcp_lp nls_utf8 isofs fuse tun ip_set nfnetlink bridge stp llc vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd iosf_mbi soundcore crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev vmw_vmci sg i2c_piix4 nfit libnvdimm parport_pc parport shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata crct10dif_pclmul crct10dif_common crc32c_intel mptspi scsi_transport_spi mptscsih serio_raw e1000 mptbase i2c_core dm_mirror dm_region_hash
dm_log dm_mod [last unloaded: ip_tables]
CPU: 7 PID: 43 Comm: ksoftirqd/7 Tainted: G        W    L ------------   3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff88017cbd8fd0 ti: ffff88017cbe8000 task.ti: ffff88017cbe8000
RIP: 0010:[<ffffffffc00503be>]  [<ffffffffc00503be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff88017cbebaa0  EFLAGS: 00000246
RAX: ffffc90000d00000 RBX: ffff8801e063d700 RCX: 0000000000000009
RDX: ffff88023736ea00 RSI: ffff8802310c8000 RDI: 000000000000000e
RBP: ffff88017cbebab0 R08: 0000000000000003 R09: ffff8802372af0d8
R10: ffff88022f3001c0 R11: ffffea0008bfb400 R12: 0000000100000001
R13: ffff88022822a100 R14: ffff88017cbeba14 R15: ffffffff812ff13b
FS:  0000000000000000(0000) GS:ffff8802357c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff6538e8078 CR3: 00000000019f2000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff8802310c8d30 000000000000003c ffff88017cbebb58 ffffffffc009e729
ffff8802310c8008 0000000000000092 0000000000000009 ffff88022f3001c0
0000000000000024 ffff8802310c8188 0000006000000009 ffff88022ff02b90
Call Trace:
[<ffffffffc009e729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffc006b2b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff8146540a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8146e1cf>] scsi_request_fn+0x4df/0x680
[<ffffffff810ba588>] ? __wake_up_common+0x58/0x90
[<ffffffff812f71d3>] __blk_run_queue+0x33/0x40
[<ffffffff812f7246>] blk_run_queue+0x26/0x40
[<ffffffff8146c6e8>] scsi_run_queue+0x258/0x2f0
[<ffffffff8146e3b0>] scsi_next_command+0x20/0x40
[<ffffffff8146e505>] scsi_end_request+0x135/0x1e0
[<ffffffff8146e778>] scsi_io_completion+0x168/0x6a0
[<ffffffff81464425>] scsi_finish_command+0xd5/0x130
[<ffffffff8146dcc2>] scsi_softirq_done+0x132/0x160
[<ffffffff81300510>] blk_done_softirq+0x90/0xc0
[<ffffffff81090b3f>] __do_softirq+0xef/0x280
[<ffffffff81090d08>] run_ksoftirqd+0x38/0x50
[<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 e3 05 
Complete report #237120
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc02123be
Build id: 3.10.0-1062.18.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc023471e
Build id: 3.10.0-1062.18.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc01562e0
Build id: 3.10.0-1062.18.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa66decd0
Build id: 3.10.0-1062.18.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffa66e833c
Build id: 3.10.0-1062.18.1.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffffa654f289
Build id: 3.10.0-1062.18.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffa654f306
Build id: 3.10.0-1062.18.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffa66e68e8
Build id: 3.10.0-1062.18.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffa66e8530
Build id: 3.10.0-1062.18.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffa66e8685
Build id: 3.10.0-1062.18.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffa66e88f8
Build id: 3.10.0-1062.18.1.el7.x86_64
0x168
12
? fbcon_add_cursor_timer
vmlinux 0xffffffffa65ff760
Build id: 3.10.0-1062.18.1.el7.x86_64
0xf0
13
scsi_finish_command
vmlinux 0xffffffffa66ddcdc
Build id: 3.10.0-1062.18.1.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffffa66e7e42
Build id: 3.10.0-1062.18.1.el7.x86_64
0x132
15
blk_done_softirq
vmlinux 0xffffffffa6558f96
Build id: 3.10.0-1062.18.1.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffffa62a5435
Build id: 3.10.0-1062.18.1.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffffa62a55f8
Build id: 3.10.0-1062.18.1.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffffa62ced54
Build id: 3.10.0-1062.18.1.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffffa62cec10
Build id: 3.10.0-1062.18.1.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffffa62c6321
Build id: 3.10.0-1062.18.1.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffffa62c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffffa698dd37
Build id: 3.10.0-1062.18.1.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffffa62c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 21s! [ksoftirqd/5:34]
Modules linked in: ip_vs_sh ip_vs_pe_sip nf_conntrack_sip ip_vs_wlc ip_vs_rr ip_vs veth nfnetlink_queue nfnetlink_log bluetooth rfkill nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) vfio_pci vfio_iommu_type1 vfio irqbypass xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd sg pcspkr joydev parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic
pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi scsi_transport_spi mptscsih mptbase drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix libata serio_raw e1000 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 5 PID: 34 Comm: ksoftirqd/5 Kdump: loaded Tainted: G        W    L ------------ T 3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff89ee7adc9070 ti: ffff89ee7add4000 task.ti: ffff89ee7add4000
RIP: 0010:[<ffffffffc02123be>]  [<ffffffffc02123be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff89ee7add7aa0  EFLAGS: 00000246
RAX: ffffad9f419c0000 RBX: ffff89ef1b89e620 RCX: 0000000000000035
RDX: ffff89ed367c7200 RSI: ffff89ed367f1000 RDI: 000000000000000e
RBP: ffff89ee7add7ab0 R08: 0000000000000003 R09: 00000002d6976fff
R10: ffff89ed367f30d8 R11: ffff89ef6da2a840 R12: 0000000100000002
R13: ffff89ef2fce6880 R14: ffff89ee7add7a14 R15: ffffffffa6557aeb
FS:  0000000000000000(0000) GS:ffff89f032f40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000416960 CR3: 000000004ea10000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc023471e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01562e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa66decd0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa66e833c>] scsi_request_fn+0x4cc/0x680
[<ffffffffa654f289>] __blk_run_queue+0x39/0x50
[<ffffffffa654f306>] blk_run_queue+0x26/0x40
[<ffffffffa66e68e8>] scsi_run_queue+0x258/0x2f0
[<ffffffffa66e8530>] scsi_next_command+0x20/0x40
[<ffffffffa66e8685>] scsi_end_request+0x135/0x1e0
[<ffffffffa66e88f8>] scsi_io_completion+0x168/0x6a0
[<ffffffffa65ff760>] ? fbcon_add_cursor_timer+0xf0/0xf0
[<ffffffffa66ddcdc>] scsi_finish_command+0xdc/0x140
[<ffffffffa66e7e42>] scsi_softirq_done+0x132/0x160
[<ffffffffa6558f96>] blk_done_softirq+0x96/0xc0
[<ffffffffa62a5435>] __do_softirq+0xf5/0x280
[<ffffffffa62a55f8>] run_ksoftirqd+0x38/0x50
[<ffffffffa62ced54>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa62cec10>] ? lg_double_unlock+0x40/0x40
[<ffffffffa62c6321>] kthread+0xd1/0xe0
[<ffffffffa62c6250>] ? insert_kthread_work+0x40/0x40
[<ffffffffa698dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa62c6250>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 03 22 
Complete report #229966
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc04873be
Build id: 3.10.0-862.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc04d671e
Build id: 3.10.0-862.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03e72e0
Build id: 3.10.0-862.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff9fe9b050
Build id: 3.10.0-862.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff9fea45bc
Build id: 3.10.0-862.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffff9fd18ad9
Build id: 3.10.0-862.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff9fd18b56
Build id: 3.10.0-862.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff9fea2b58
Build id: 3.10.0-862.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff9fea47b0
Build id: 3.10.0-862.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff9fea4905
Build id: 3.10.0-862.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff9fea4b78
Build id: 3.10.0-862.el7.x86_64
0x168
12
scsi_finish_command
vmlinux 0xffffffff9fe9a05c
Build id: 3.10.0-862.el7.x86_64
0xdc
13
scsi_softirq_done
vmlinux 0xffffffff9fea40c2
Build id: 3.10.0-862.el7.x86_64
0x132
14
blk_done_softirq
vmlinux 0xffffffff9fd22796
Build id: 3.10.0-862.el7.x86_64
0x96
15
__do_softirq
vmlinux 0xffffffff9fa9a945
Build id: 3.10.0-862.el7.x86_64
0xf5
16
run_ksoftirqd
vmlinux 0xffffffff9fa9ab08
Build id: 3.10.0-862.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffff9fac35e4
Build id: 3.10.0-862.el7.x86_64
0x144
18
? lg_double_unlock
vmlinux 0xffffffff9fac34a0
Build id: 3.10.0-862.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffff9fabae31
Build id: 3.10.0-862.el7.x86_64
0xd1
20
? insert_kthread_work
vmlinux 0xffffffff9fabad60
Build id: 3.10.0-862.el7.x86_64
0x40
21
ret_from_fork_nospec_begin
vmlinux 0xffffffffa011f637
Build id: 3.10.0-862.el7.x86_64
0x21
22
? insert_kthread_work
vmlinux 0xffffffff9fabad60
Build id: 3.10.0-862.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 28s! [ksoftirqd/2:19]
Modules linked in: esn_cfs(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm coretemp snd_timer snd soundcore iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_balloon parport_pc parport shpchp vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul
syscopyarea crct10dif_common sysfillrect sysimgblt fb_sys_fops crc32c_intel ttm ata_piix drm mptspi serio_raw scsi_transport_spi mptscsih e1000 mptbase libata i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: G        W  OEL ------------   3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2012
task: ffff8870bb068fd0 ti: ffff8870bb070000 task.ti: ffff8870bb070000
RIP: 0010:[<ffffffffc04873be>]  [<ffffffffc04873be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8870bb073aa0  EFLAGS: 00000246
RAX: ffffb42600380000 RBX: ffff887064c7ee20 RCX: 0000000000000009
RDX: ffff887075b7a200 RSI: ffff887075b88000 RDI: 000000000000000e
RBP: ffff8870bb073ab0 R08: 0000000000000002 R09: 0000000021070fff
R10: ffff887075df10d8 R11: ffffdcbb00d6df00 R12: 0000000100000002
R13: ffff887064f8f780 R14: ffff8870bb073a14 R15: ffffffff9fd212cb
FS:  0000000000000000(0000) GS:ffff8870bc680000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0535b39000 CR3: 0000000078fe4000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc04d671e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03e72e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9fe9b050>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9fea45bc>] scsi_request_fn+0x4cc/0x680
[<ffffffff9fd18ad9>] __blk_run_queue+0x39/0x50
[<ffffffff9fd18b56>] blk_run_queue+0x26/0x40
[<ffffffff9fea2b58>] scsi_run_queue+0x258/0x2f0
[<ffffffff9fea47b0>] scsi_next_command+0x20/0x40
[<ffffffff9fea4905>] scsi_end_request+0x135/0x1e0
[<ffffffff9fea4b78>] scsi_io_completion+0x168/0x6a0
[<ffffffff9fe9a05c>] scsi_finish_command+0xdc/0x140
[<ffffffff9fea40c2>] scsi_softirq_done+0x132/0x160
[<ffffffff9fd22796>] blk_done_softirq+0x96/0xc0
[<ffffffff9fa9a945>] __do_softirq+0xf5/0x280
[<ffffffff9fa9ab08>] run_ksoftirqd+0x38/0x50
[<ffffffff9fac35e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9fac34a0>] ? lg_double_unlock+0x40/0x40
[<ffffffff9fabae31>] kthread+0xd1/0xe0
[<ffffffff9fabad60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa011f637>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9fabad60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 49 
Complete report #210030
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01e23be
Build id: 3.10.0-1160.15.2.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc023171e
Build id: 3.10.0-1160.15.2.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc023c2e0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa36e3e70
Build id: 3.10.0-1160.15.2.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffa36ed59c
Build id: 3.10.0-1160.15.2.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffa3553869
Build id: 3.10.0-1160.15.2.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffa35538e6
Build id: 3.10.0-1160.15.2.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffa36ebaf8
Build id: 3.10.0-1160.15.2.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffa36ed7b0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffa36ed905
Build id: 3.10.0-1160.15.2.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffa36edb78
Build id: 3.10.0-1160.15.2.el7.x86_64
0x168
12
? __schedule
vmlinux 0xffffffffa3987af2
Build id: 3.10.0-1160.15.2.el7.x86_64
0x5a2
13
? __schedule
vmlinux 0xffffffffa3987ae6
Build id: 3.10.0-1160.15.2.el7.x86_64
0x596
14
scsi_finish_command
vmlinux 0xffffffffa36e2e6c
Build id: 3.10.0-1160.15.2.el7.x86_64
0xdc
15
scsi_softirq_done
vmlinux 0xffffffffa36ed0c0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x130
16
blk_done_softirq
vmlinux 0xffffffffa355d6c6
Build id: 3.10.0-1160.15.2.el7.x86_64
0x96
17
__do_softirq
vmlinux 0xffffffffa32a4b35
Build id: 3.10.0-1160.15.2.el7.x86_64
0xf5
18
run_ksoftirqd
vmlinux 0xffffffffa32a4cf8
Build id: 3.10.0-1160.15.2.el7.x86_64
0x38
19
smpboot_thread_fn
vmlinux 0xffffffffa32ce864
Build id: 3.10.0-1160.15.2.el7.x86_64
0x144
20
? lg_double_unlock
vmlinux 0xffffffffa32ce720
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
21
kthread
vmlinux 0xffffffffa32c5e11
Build id: 3.10.0-1160.15.2.el7.x86_64
0xd1
22
? insert_kthread_work
vmlinux 0xffffffffa32c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
23
ret_from_fork_nospec_begin
vmlinux 0xffffffffa3994df7
Build id: 3.10.0-1160.15.2.el7.x86_64
0x21
24
? insert_kthread_work
vmlinux 0xffffffffa32c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ksoftirqd/5:34]
Modules linked in: tcp_diag inet_diag rfcomm ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack talpa_vfshook(OE) talpa_pedconnector(OE) talpa_pedevice(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq iosf_mbi snd_seq_device crc32_pclmul snd_pcm ghash_clmulni_intel
btusb btrtl btbcm btintel snd_timer bluetooth snd aesni_intel ppdev lrw gf128mul glue_helper soundcore ablk_helper rfkill sg cryptd joydev pcspkr vmw_balloon parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit crct10dif_pclmul drm crct10dif_common ata_piix crc32c_intel libnvdimm libata mptspi scsi_transport_spi serio_raw mptscsih e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 5 PID: 34 Comm: ksoftirqd/5 Kdump: loaded Tainted: G           OEL ------------   3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff98e12fb9e300 ti: ffff98e12fbc0000 task.ti: ffff98e12fbc0000
RIP: 0010:[<ffffffffc01e23be>]  [<ffffffffc01e23be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff98e12fbc3aa0  EFLAGS: 00000246
RAX: ffffb9a741940000 RBX: ffff98e14d1ef5a0 RCX: 0000000000000017
RDX: ffff98e28ec35000 RSI: ffff98e29020b000 RDI: 000000000000000e
RBP: ffff98e12fbc3ab0 R08: 0000000000000003 R09: 00000002152a7fff
R10: 000000008e063401 R11: ffff98e28e063640 R12: 0000000100000006
R13: ffff98e146721500 R14: ffff98e12fbc3a14 R15: ffffffffa355c28b
FS:  0000000000000000(0000) GS:ffff98e292f40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2725271000 CR3: 000000014c810000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc023171e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc023c2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa36e3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa36ed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffffa3553869>] __blk_run_queue+0x39/0x50
[<ffffffffa35538e6>] blk_run_queue+0x26/0x40
[<ffffffffa36ebaf8>] scsi_run_queue+0x258/0x2f0
[<ffffffffa36ed7b0>] scsi_next_command+0x20/0x40
[<ffffffffa36ed905>] scsi_end_request+0x135/0x1e0
[<ffffffffa36edb78>] scsi_io_completion+0x168/0x720
[<ffffffffa3987af2>] ? __schedule+0x5a2/0x860
[<ffffffffa3987ae6>] ? __schedule+0x596/0x860
[<ffffffffa36e2e6c>] scsi_finish_command+0xdc/0x140
[<ffffffffa36ed0c0>] scsi_softirq_done+0x130/0x160
[<ffffffffa355d6c6>] blk_done_softirq+0x96/0xc0
[<ffffffffa32a4b35>] __do_softirq+0xf5/0x280
[<ffffffffa32a4cf8>] run_ksoftirqd+0x38/0x50
[<ffffffffa32ce864>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa32ce720>] ? lg_double_unlock+0x40/0x40
[<ffffffffa32c5e11>] kthread+0xd1/0xe0
[<ffffffffa32c5d40>] ? insert_kthread_work+0x40/0x40
[<ffffffffa3994df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa32c5d40>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 03 1f 
Complete report #170253
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffa007b3be
Build id: 3.10.0-327.28.3.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffa00a8729
Build id: 3.10.0-327.28.3.el7.x86_64
0x249
3
mptspi_qcmd
mptspi 0xffffffffa00962b0
Build id: 3.10.0-327.28.3.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8141845a
Build id: 3.10.0-327.28.3.el7.x86_64
0xaa
5
scsi_request_fn
vmlinux 0xffffffff814213e1
Build id: 3.10.0-327.28.3.el7.x86_64
0x501
6
__blk_run_queue
vmlinux 0xffffffff812c7c63
Build id: 3.10.0-327.28.3.el7.x86_64
0x33
7
blk_run_queue
vmlinux 0xffffffff812c7cd6
Build id: 3.10.0-327.28.3.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff8141f808
Build id: 3.10.0-327.28.3.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff81421690
Build id: 3.10.0-327.28.3.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff81421805
Build id: 3.10.0-327.28.3.el7.x86_64
0x155
11
scsi_io_completion
vmlinux 0xffffffff814219e3
Build id: 3.10.0-327.28.3.el7.x86_64
0x103
12
? fbcon_add_cursor_timer
vmlinux 0xffffffff8134fcd0
Build id: 3.10.0-327.28.3.el7.x86_64
0xf0
13
scsi_finish_command
vmlinux 0xffffffff81416d25
Build id: 3.10.0-327.28.3.el7.x86_64
0xd5
14
scsi_softirq_done
vmlinux 0xffffffff81420eba
Build id: 3.10.0-327.28.3.el7.x86_64
0x12a
15
blk_done_softirq
vmlinux 0xffffffff812d1f80
Build id: 3.10.0-327.28.3.el7.x86_64
0x90
16
__do_softirq
vmlinux 0xffffffff81084b4f
Build id: 3.10.0-327.28.3.el7.x86_64
0xef
17
run_ksoftirqd
vmlinux 0xffffffff81084d10
Build id: 3.10.0-327.28.3.el7.x86_64
0x30
18
smpboot_thread_fn
vmlinux 0xffffffff810ae58f
Build id: 3.10.0-327.28.3.el7.x86_64
0xff
19
? schedule
vmlinux 0xffffffff8163bb59
Build id: 3.10.0-327.28.3.el7.x86_64
0x29
20
? lg_double_unlock
vmlinux 0xffffffff810ae490
Build id: 3.10.0-327.28.3.el7.x86_64
0x90
21
kthread
vmlinux 0xffffffff810a5b2f
Build id: 3.10.0-327.28.3.el7.x86_64
0xcf
22
? kthread_create_on_node
vmlinux 0xffffffff810a5a60
Build id: 3.10.0-327.28.3.el7.x86_64
0x140
23
ret_from_fork
vmlinux 0xffffffff81646a98
Build id: 3.10.0-327.28.3.el7.x86_64
0x58
24
? kthread_create_on_node
vmlinux 0xffffffff810a5a60
Build id: 3.10.0-327.28.3.el7.x86_64
0x140
BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:3]
Modules linked in: binfmt_misc vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr sg vmw_vmci i2c_piix4 shpchp parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common drm_kms_helper crc32c_intel serio_raw ttm drm ata_piix mptspi scsi_transport_spi mptscsih i2c_core mptbase libata vmxnet3 floppy
CPU: 0 PID: 3 Comm: ksoftirqd/0 Not tainted 3.10.0-327.28.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/17/2015
task: ffff880234161700 ti: ffff880234188000 task.ti: ffff880234188000
RIP: 0010:[<ffffffffa007b3be>]  [<ffffffffa007b3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88023418ba80  EFLAGS: 00000246
RAX: ffffc90008f40000 RBX: ffff88023418ba50 RCX: 000000000000000c
RDX: ffff880036b15a00 RSI: ffff880233e32000 RDI: 000000000000000e
RBP: ffff88023418ba90 R08: 0000000000000003 R09: ffff8802300050d8
R10: ffff88004c382000 R11: ffffea0007f39d80 R12: 0000000100000002
R13: ffff8800afc87980 R14: ffff88023418b9f4 R15: ffffffff812d0f9b
FS:  0000000000000000(0000) GS:ffff88023fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fe07b435000 CR3: 0000000062495000 CR4: 00000000000407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880233e32d30 0000000000000048 ffff88023418bb38 ffffffffa00a8729
ffff880233e32008 040000003418baf8 000000000000000c ffff88004c382000
0000000000000030 ffff880233e32188 000000600000000c ffff880230142cb0
Call Trace:
[<ffffffffa00a8729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa00962b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff8141845a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff814213e1>] scsi_request_fn+0x501/0x770
[<ffffffff812c7c63>] __blk_run_queue+0x33/0x40
[<ffffffff812c7cd6>] blk_run_queue+0x26/0x40
[<ffffffff8141f808>] scsi_run_queue+0x258/0x2f0
[<ffffffff81421690>] scsi_next_command+0x20/0x40
[<ffffffff81421805>] scsi_end_request+0x155/0x1d0
[<ffffffff814219e3>] scsi_io_completion+0x103/0x600
[<ffffffff8134fcd0>] ? fbcon_add_cursor_timer+0xf0/0xf0
[<ffffffff81416d25>] scsi_finish_command+0xd5/0x130
[<ffffffff81420eba>] scsi_softirq_done+0x12a/0x150
[<ffffffff812d1f80>] blk_done_softirq+0x90/0xc0
[<ffffffff81084b4f>] __do_softirq+0xef/0x280
[<ffffffff81084d10>] run_ksoftirqd+0x30/0x50
[<ffffffff810ae58f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163bb59>] ? schedule+0x29/0x70
[<ffffffff810ae490>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5b2f>] kthread+0xcf/0xe0
[<ffffffff810a5a60>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81646a98>] ret_from_fork+0x58/0x90
[<ffffffff810a5a60>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 93 08 
Complete report #161285
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03df3be
Build id: 3.10.0-1160.25.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc04a871e
Build id: 3.10.0-1160.25.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc055f2e0
Build id: 3.10.0-1160.25.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffbbee3fa0
Build id: 3.10.0-1160.25.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffbbeed6cc
Build id: 3.10.0-1160.25.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffbbd53939
Build id: 3.10.0-1160.25.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffbbd539b6
Build id: 3.10.0-1160.25.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffbbeebc28
Build id: 3.10.0-1160.25.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffbbeed8e0
Build id: 3.10.0-1160.25.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffbbeeda35
Build id: 3.10.0-1160.25.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffbbeedca8
Build id: 3.10.0-1160.25.1.el7.x86_64
0x168
12
? __schedule
vmlinux 0xffffffffbc188c62
Build id: 3.10.0-1160.25.1.el7.x86_64
0x5b2
13
? __schedule
vmlinux 0xffffffffbc188c56
Build id: 3.10.0-1160.25.1.el7.x86_64
0x5a6
14
scsi_finish_command
vmlinux 0xffffffffbbee2f9c
Build id: 3.10.0-1160.25.1.el7.x86_64
0xdc
15
scsi_softirq_done
vmlinux 0xffffffffbbeed1f0
Build id: 3.10.0-1160.25.1.el7.x86_64
0x130
16
blk_done_softirq
vmlinux 0xffffffffbbd5d796
Build id: 3.10.0-1160.25.1.el7.x86_64
0x96
17
__do_softirq
vmlinux 0xffffffffbbaa4b35
Build id: 3.10.0-1160.25.1.el7.x86_64
0xf5
18
run_ksoftirqd
vmlinux 0xffffffffbbaa4cf8
Build id: 3.10.0-1160.25.1.el7.x86_64
0x38
19
smpboot_thread_fn
vmlinux 0xffffffffbbace7f4
Build id: 3.10.0-1160.25.1.el7.x86_64
0x144
20
? lg_double_unlock
vmlinux 0xffffffffbbace6b0
Build id: 3.10.0-1160.25.1.el7.x86_64
0x40
21
kthread
vmlinux 0xffffffffbbac5da1
Build id: 3.10.0-1160.25.1.el7.x86_64
0xd1
22
? insert_kthread_work
vmlinux 0xffffffffbbac5cd0
Build id: 3.10.0-1160.25.1.el7.x86_64
0x40
23
ret_from_fork_nospec_begin
vmlinux 0xffffffffbc195df7
Build id: 3.10.0-1160.25.1.el7.x86_64
0x21
24
? insert_kthread_work
vmlinux 0xffffffffbbac5cd0
Build id: 3.10.0-1160.25.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) bnep vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel btusb btrtl btbcm aesni_intel btintel bluetooth ac97_bus lrw snd_seq gf128mul glue_helper ablk_helper cryptd snd_seq_device vmw_balloon snd_pcm pcspkr joydev snd_timer rfkill snd sg vmw_vmci i2c_piix4 soundcore parport_pc
parport sch_fq_codel ip_tables ext4 mbcache jbd2 vmwgfx sd_mod sr_mod crc_t10dif crct10dif_generic cdrom ata_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common crc32c_intel pata_acpi ttm mptspi scsi_transport_spi drm mptscsih nfit ata_piix libnvdimm libata serio_raw e1000 mptbase drm_panel_orientation_quirks fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Tainted: G             L ------------ T 3.10.0-1160.25.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff899c7a3cd280 ti: ffff899c7af80000 task.ti: ffff899c7af80000
RIP: 0010:[<ffffffffc03df3be>]  [<ffffffffc03df3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff899c7af83aa0  EFLAGS: 00000246
RAX: ffffb7d3c0240000 RBX: ffffffffbbad7422 RCX: 000000000000001c
RDX: ffff899c764e4000 RSI: ffff899c75848000 RDI: 000000000000000e
RBP: ffff899c7af83ab0 R08: 0000000000000003 R09: 00000000108a5fff
R10: ffff899c7c623b90 R11: ffffe314805a9400 R12: 0000000000000000
R13: ffff899c7c61acc0 R14: ffff899c7c61acc0 R15: ffff899c54f86300
FS:  0000000000000000(0000) GS:ffff899c7c600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000045a970 CR3: 00000000387cc000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc04a871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc055f2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffbbee3fa0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffbbeed6cc>] scsi_request_fn+0x4ac/0x680
[<ffffffffbbd53939>] __blk_run_queue+0x39/0x50
[<ffffffffbbd539b6>] blk_run_queue+0x26/0x40
[<ffffffffbbeebc28>] scsi_run_queue+0x258/0x2f0
[<ffffffffbbeed8e0>] scsi_next_command+0x20/0x40
[<ffffffffbbeeda35>] scsi_end_request+0x135/0x1e0
[<ffffffffbbeedca8>] scsi_io_completion+0x168/0x720
[<ffffffffbc188c62>] ? __schedule+0x5b2/0x870
[<ffffffffbc188c56>] ? __schedule+0x5a6/0x870
[<ffffffffbbee2f9c>] scsi_finish_command+0xdc/0x140
[<ffffffffbbeed1f0>] scsi_softirq_done+0x130/0x160
[<ffffffffbbd5d796>] blk_done_softirq+0x96/0xc0
[<ffffffffbbaa4b35>] __do_softirq+0xf5/0x280
[<ffffffffbbaa4cf8>] run_ksoftirqd+0x38/0x50
[<ffffffffbbace7f4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffbbace6b0>] ? lg_double_unlock+0x40/0x40
[<ffffffffbbac5da1>] kthread+0xd1/0xe0
[<ffffffffbbac5cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffffbc195df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffbbac5cd0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 d3 3e 
Complete report #142616
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03253be
Build id: 3.10.0-1160.6.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc02f671e
Build id: 3.10.0-1160.6.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc02ef2e0
Build id: 3.10.0-1160.6.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa14e39d0
Build id: 3.10.0-1160.6.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffa14ed0fc
Build id: 3.10.0-1160.6.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffa1353519
Build id: 3.10.0-1160.6.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffa1353596
Build id: 3.10.0-1160.6.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffa14eb658
Build id: 3.10.0-1160.6.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffa14ed310
Build id: 3.10.0-1160.6.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffa14ed465
Build id: 3.10.0-1160.6.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffa14ed6d8
Build id: 3.10.0-1160.6.1.el7.x86_64
0x168
12
? scsi_io_completion
vmlinux 0xffffffffa14ed6b8
Build id: 3.10.0-1160.6.1.el7.x86_64
0x148
13
scsi_finish_command
vmlinux 0xffffffffa14e29cc
Build id: 3.10.0-1160.6.1.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffffa14ecc20
Build id: 3.10.0-1160.6.1.el7.x86_64
0x130
15
blk_done_softirq
vmlinux 0xffffffffa135d266
Build id: 3.10.0-1160.6.1.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffffa10a4b95
Build id: 3.10.0-1160.6.1.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffffa10a4d58
Build id: 3.10.0-1160.6.1.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffffa10ce674
Build id: 3.10.0-1160.6.1.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffffa10ce530
Build id: 3.10.0-1160.6.1.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffffa10c5c21
Build id: 3.10.0-1160.6.1.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffffa10c5b50
Build id: 3.10.0-1160.6.1.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffffa1794df7
Build id: 3.10.0-1160.6.1.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffffa10c5b50
Build id: 3.10.0-1160.6.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 21s! [ksoftirqd/6:39]
Modules linked in: sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev sg parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm e1000 ata_piix crct10dif_pclmul crct10dif_common crc32c_intel libata serio_raw mptspi drm scsi_transport_spi vmxnet3 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 39 Comm: ksoftirqd/6 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff9bd37eb43180 ti: ffff9bd37eb50000 task.ti: ffff9bd37eb50000
RIP: 0010:[<ffffffffc03253be>]  [<ffffffffc03253be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff9bd37eb53aa0  EFLAGS: 00000246
RAX: ffffb38180ec0000 RBX: ffff9bd35e7a0fe0 RCX: 0000000000000017
RDX: ffff9bd23657f400 RSI: ffff9bd435ada000 RDI: 000000000000000e
RBP: ffff9bd37eb53ab0 R08: 0000000000000003 R09: 0000000090887fff
R10: 0000000097a24f01 R11: ffff9bd397a25c00 R12: 0000000100000080
R13: ffff9bd25c6be780 R14: ffff9bd37eb53a14 R15: ffffffffa135be3b
FS:  0000000000000000(0000) GS:ffff9bd43fd80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000080055cf78 CR3: 000000016ea10000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc02f671e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02ef2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa14e39d0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa14ed0fc>] scsi_request_fn+0x4ac/0x680
[<ffffffffa1353519>] __blk_run_queue+0x39/0x50
[<ffffffffa1353596>] blk_run_queue+0x26/0x40
[<ffffffffa14eb658>] scsi_run_queue+0x258/0x2f0
[<ffffffffa14ed310>] scsi_next_command+0x20/0x40
[<ffffffffa14ed465>] scsi_end_request+0x135/0x1e0
[<ffffffffa14ed6d8>] scsi_io_completion+0x168/0x720
[<ffffffffa14ed6b8>] ? scsi_io_completion+0x148/0x720
[<ffffffffa14e29cc>] scsi_finish_command+0xdc/0x140
[<ffffffffa14ecc20>] scsi_softirq_done+0x130/0x160
[<ffffffffa135d266>] blk_done_softirq+0x96/0xc0
[<ffffffffa10a4b95>] __do_softirq+0xf5/0x280
[<ffffffffa10a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffa10ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa10ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffffa10c5c21>] kthread+0xd1/0xe0
[<ffffffffa10c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffa1794df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa10c5b50>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 33 33 
Complete report #125560
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffbb1883c5
Build id: 3.10.0-1127.19.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc005d632
Build id: 3.10.0-1127.19.1.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffffbaeeb0a0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffffbaee2990
Build id: 3.10.0-1127.19.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffbaeec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffbad527a9
Build id: 3.10.0-1127.19.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffbad52826
Build id: 3.10.0-1127.19.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffbaeea618
Build id: 3.10.0-1127.19.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffbaeec2d0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffbaeec425
Build id: 3.10.0-1127.19.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffbaeec698
Build id: 3.10.0-1127.19.1.el7.x86_64
0x168
12
? __schedule
vmlinux 0xffffffffbb185850
Build id: 3.10.0-1127.19.1.el7.x86_64
0x310
13
? __schedule
vmlinux 0xffffffffbb185844
Build id: 3.10.0-1127.19.1.el7.x86_64
0x304
14
scsi_finish_command
vmlinux 0xffffffffbaee198c
Build id: 3.10.0-1127.19.1.el7.x86_64
0xdc
15
scsi_softirq_done
vmlinux 0xffffffffbaeebbe0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x130
16
blk_done_softirq
vmlinux 0xffffffffbad5c4f6
Build id: 3.10.0-1127.19.1.el7.x86_64
0x96
17
__do_softirq
vmlinux 0xffffffffbaaa5695
Build id: 3.10.0-1127.19.1.el7.x86_64
0xf5
18
run_ksoftirqd
vmlinux 0xffffffffbaaa5858
Build id: 3.10.0-1127.19.1.el7.x86_64
0x38
19
smpboot_thread_fn
vmlinux 0xffffffffbaacf0e4
Build id: 3.10.0-1127.19.1.el7.x86_64
0x144
20
? lg_double_unlock
vmlinux 0xffffffffbaacefa0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
21
kthread
vmlinux 0xffffffffbaac6691
Build id: 3.10.0-1127.19.1.el7.x86_64
0xd1
22
? insert_kthread_work
vmlinux 0xffffffffbaac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
23
ret_from_fork_nospec_begin
vmlinux 0xffffffffbb192d37
Build id: 3.10.0-1127.19.1.el7.x86_64
0x21
24
? insert_kthread_work
vmlinux 0xffffffffbaac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [ksoftirqd/0:6]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter ipmi_devintf ipmi_msghandler vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul glue_helper ablk_helper cryptd pcspkr joydev vmw_balloon sg parport_pc parport i2c_piix4 vmw_vmci binfmt_misc ip_tables xfs libcrc32c ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix ahci
drm libahci crct10dif_pclmul crct10dif_common e1000e nfit crc32c_intel libnvdimm libata serio_raw ptp pps_core vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8e763a315230 ti: ffff8e763a34c000 task.ti: ffff8e763a34c000
RIP: 0010:[<ffffffffbb1883c5>]  [<ffffffffbb1883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8e763a34fb08  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff8e763a34fac8 RCX: ffffffffbb634e40
RDX: ffffa64701b90000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8e763a34fb08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8e78e9709c00 R11: ffff8e78e9928000 R12: 0000000000000020
R13: ffff8e74f57a3c80 R14: ffffffffbaee9d59 R15: ffff8e763a34fa80
FS:  0000000000000000(0000) GS:ffff8e78ffc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000006dee20 CR3: 00000003861c0000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc005d632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffbaeeb0a0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffbaee2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffbaeec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffffbad527a9>] __blk_run_queue+0x39/0x50
[<ffffffffbad52826>] blk_run_queue+0x26/0x40
[<ffffffffbaeea618>] scsi_run_queue+0x258/0x2f0
[<ffffffffbaeec2d0>] scsi_next_command+0x20/0x40
[<ffffffffbaeec425>] scsi_end_request+0x135/0x1e0
[<ffffffffbaeec698>] scsi_io_completion+0x168/0x720
[<ffffffffbb185850>] ? __schedule+0x310/0x840
[<ffffffffbb185844>] ? __schedule+0x304/0x840
[<ffffffffbaee198c>] scsi_finish_command+0xdc/0x140
[<ffffffffbaeebbe0>] scsi_softirq_done+0x130/0x160
[<ffffffffbad5c4f6>] blk_done_softirq+0x96/0xc0
[<ffffffffbaaa5695>] __do_softirq+0xf5/0x280
[<ffffffffbaaa5858>] run_ksoftirqd+0x38/0x50
[<ffffffffbaacf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffbaacefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffbaac6691>] kthread+0xd1/0xe0
[<ffffffffbaac65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffbb192d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffbaac65c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #101072
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03353be
Build id: 3.10.0-1062.12.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc041871e
Build id: 3.10.0-1062.12.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03762e0
Build id: 3.10.0-1062.12.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8d4deb10
Build id: 3.10.0-1062.12.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff8d4e817c
Build id: 3.10.0-1062.12.1.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffff8d34f0c9
Build id: 3.10.0-1062.12.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff8d34f146
Build id: 3.10.0-1062.12.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff8d4e6728
Build id: 3.10.0-1062.12.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff8d4e8370
Build id: 3.10.0-1062.12.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff8d4e84c5
Build id: 3.10.0-1062.12.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff8d4e8738
Build id: 3.10.0-1062.12.1.el7.x86_64
0x168
12
? put_prev_entity
vmlinux 0xffffffff8d0e652b
Build id: 3.10.0-1062.12.1.el7.x86_64
0x8b
13
scsi_finish_command
vmlinux 0xffffffff8d4ddb1c
Build id: 3.10.0-1062.12.1.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffff8d4e7c82
Build id: 3.10.0-1062.12.1.el7.x86_64
0x132
15
blk_done_softirq
vmlinux 0xffffffff8d358dd6
Build id: 3.10.0-1062.12.1.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffff8d0a5305
Build id: 3.10.0-1062.12.1.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffff8d0a54c8
Build id: 3.10.0-1062.12.1.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff8d0cec24
Build id: 3.10.0-1062.12.1.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffff8d0ceae0
Build id: 3.10.0-1062.12.1.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffff8d0c61f1
Build id: 3.10.0-1062.12.1.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffff8d0c6120
Build id: 3.10.0-1062.12.1.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffff8d78dd37
Build id: 3.10.0-1062.12.1.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffff8d0c6120
Build id: 3.10.0-1062.12.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 24s! [ksoftirqd/7:46]
Modules linked in: tcp_diag udp_diag inet_diag nf_conntrack_netlink nfnetlink nf_conntrack_ftp binfmt_misc xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 ppdev vmw_balloon sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper joydev cryptd pcspkr parport_pc sg parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic vmwgfx drm_kms_helper syscopyarea pata_acpi sysfillrect sysimgblt fb_sys_fops ttm drm
mptspi ata_piix scsi_transport_spi mptscsih crct10dif_pclmul crct10dif_common libata serio_raw vmxnet3 mptbase crc32c_intel drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 7 PID: 46 Comm: ksoftirqd/7 Kdump: loaded Tainted: G             L ------------   3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff91a8ba5b8000 ti: ffff91a8ba5b4000 task.ti: ffff91a8ba5b4000
RIP: 0010:[<ffffffffc03353be>]  [<ffffffffc03353be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff91a8ba5b7aa0  EFLAGS: 00000246
RAX: ffff9fa2c36c0000 RBX: ffff91a8ba5b7a40 RCX: 000000000000002f
RDX: ffff91a435cecc00 RSI: ffff91ac2d7bb000 RDI: 000000000000000e
RBP: ffff91a8ba5b7ab0 R08: 0000000000000003 R09: 000000027d948fff
R10: ffff91a8ba4ce0d8 R11: ffffc1ede0fe7100 R12: ffff91a8ba5b7a20
R13: ffffffff8d0635a3 R14: ffff91a8ba5b7a10 R15: 0000000000000006
FS:  0000000000000000(0000) GS:ffff91ac3fc40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f59509cf000 CR3: 00000003c3210000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc041871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03762e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8d4deb10>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8d4e817c>] scsi_request_fn+0x4cc/0x680
[<ffffffff8d34f0c9>] __blk_run_queue+0x39/0x50
[<ffffffff8d34f146>] blk_run_queue+0x26/0x40
[<ffffffff8d4e6728>] scsi_run_queue+0x258/0x2f0
[<ffffffff8d4e8370>] scsi_next_command+0x20/0x40
[<ffffffff8d4e84c5>] scsi_end_request+0x135/0x1e0
[<ffffffff8d4e8738>] scsi_io_completion+0x168/0x6a0
[<ffffffff8d0e652b>] ? put_prev_entity+0x8b/0x400
[<ffffffff8d4ddb1c>] scsi_finish_command+0xdc/0x140
[<ffffffff8d4e7c82>] scsi_softirq_done+0x132/0x160
[<ffffffff8d358dd6>] blk_done_softirq+0x96/0xc0
[<ffffffff8d0a5305>] __do_softirq+0xf5/0x280
[<ffffffff8d0a54c8>] run_ksoftirqd+0x38/0x50
[<ffffffff8d0cec24>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8d0ceae0>] ? lg_double_unlock+0x40/0x40
[<ffffffff8d0c61f1>] kthread+0xd1/0xe0
[<ffffffff8d0c6120>] ? insert_kthread_work+0x40/0x40
[<ffffffff8d78dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8d0c6120>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 33 34 
Complete report #80280
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffb71883c5
Build id: 3.10.0-1127.19.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0250632
Build id: 3.10.0-1127.19.1.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffffb6eeb0a0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffffb6ee2990
Build id: 3.10.0-1127.19.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb6eec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffb6d527a9
Build id: 3.10.0-1127.19.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffb6d52826
Build id: 3.10.0-1127.19.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffb6eea618
Build id: 3.10.0-1127.19.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffb6eec2d0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffb6eec425
Build id: 3.10.0-1127.19.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffb6eec698
Build id: 3.10.0-1127.19.1.el7.x86_64
0x168
12
scsi_finish_command
vmlinux 0xffffffffb6ee198c
Build id: 3.10.0-1127.19.1.el7.x86_64
0xdc
13
scsi_softirq_done
vmlinux 0xffffffffb6eebbe0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x130
14
blk_done_softirq
vmlinux 0xffffffffb6d5c4f6
Build id: 3.10.0-1127.19.1.el7.x86_64
0x96
15
__do_softirq
vmlinux 0xffffffffb6aa5695
Build id: 3.10.0-1127.19.1.el7.x86_64
0xf5
16
run_ksoftirqd
vmlinux 0xffffffffb6aa5858
Build id: 3.10.0-1127.19.1.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffffb6acf0e4
Build id: 3.10.0-1127.19.1.el7.x86_64
0x144
18
? lg_double_unlock
vmlinux 0xffffffffb6acefa0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffffb6ac6691
Build id: 3.10.0-1127.19.1.el7.x86_64
0xd1
20
? insert_kthread_work
vmlinux 0xffffffffb6ac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
21
ret_from_fork_nospec_begin
vmlinux 0xffffffffb7192d37
Build id: 3.10.0-1127.19.1.el7.x86_64
0x21
22
? insert_kthread_work
vmlinux 0xffffffffb6ac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 24s! [ksoftirqd/3:24]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink ipmi_devintf ipmi_msghandler vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg ppdev pcspkr vmw_balloon parport_pc joydev parport vmw_vmci i2c_piix4 binfmt_misc ip_tables xfs libcrc32c ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci e1000e nfit
libnvdimm drm libahci ata_piix crct10dif_pclmul crct10dif_common libata crc32c_intel serio_raw vmw_pvscsi ptp pps_core drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffffa0f9344d0000 ti: ffffa0f9344d8000 task.ti: ffffa0f9344d8000
RIP: 0010:[<ffffffffb71883c5>]  [<ffffffffb71883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffffa0f9344dbb08  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffa0f9344dbac8 RCX: ffffffffb7634e40
RDX: ffffc09b035b0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffffa0f9344dbb08 R08: 0000000000000001 R09: 0000000000000000
R10: ffffa0ffd94c6000 R11: ffffa0ffda0dd800 R12: 0000000000000020
R13: ffffa0f857b48300 R14: ffffffffb6ee9d59 R15: ffffa0f9344dba80
FS:  0000000000000000(0000) GS:ffffa0ffffcc0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000105b6f0 CR3: 000000081e1d4000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc0250632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffb6eeb0a0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffb6ee2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb6eec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffffb6d527a9>] __blk_run_queue+0x39/0x50
[<ffffffffb6d52826>] blk_run_queue+0x26/0x40
[<ffffffffb6eea618>] scsi_run_queue+0x258/0x2f0
[<ffffffffb6eec2d0>] scsi_next_command+0x20/0x40
[<ffffffffb6eec425>] scsi_end_request+0x135/0x1e0
[<ffffffffb6eec698>] scsi_io_completion+0x168/0x720
[<ffffffffb6ee198c>] scsi_finish_command+0xdc/0x140
[<ffffffffb6eebbe0>] scsi_softirq_done+0x130/0x160
[<ffffffffb6d5c4f6>] blk_done_softirq+0x96/0xc0
[<ffffffffb6aa5695>] __do_softirq+0xf5/0x280
[<ffffffffb6aa5858>] run_ksoftirqd+0x38/0x50
[<ffffffffb6acf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb6acefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb6ac6691>] kthread+0xd1/0xe0
[<ffffffffb6ac65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb7192d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb6ac65c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #67637
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff8cb89845
Build id: 3.10.0-1160.6.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc03c7632
Build id: 3.10.0-1160.6.1.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff8c8ec0e0
Build id: 3.10.0-1160.6.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff8c8e39d0
Build id: 3.10.0-1160.6.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff8c8ed0fc
Build id: 3.10.0-1160.6.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffff8c753519
Build id: 3.10.0-1160.6.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff8c753596
Build id: 3.10.0-1160.6.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff8c8eb658
Build id: 3.10.0-1160.6.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff8c8ed310
Build id: 3.10.0-1160.6.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff8c8ed465
Build id: 3.10.0-1160.6.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff8c8ed6d8
Build id: 3.10.0-1160.6.1.el7.x86_64
0x168
12
scsi_finish_command
vmlinux 0xffffffff8c8e29cc
Build id: 3.10.0-1160.6.1.el7.x86_64
0xdc
13
scsi_softirq_done
vmlinux 0xffffffff8c8ecc20
Build id: 3.10.0-1160.6.1.el7.x86_64
0x130
14
blk_done_softirq
vmlinux 0xffffffff8c75d266
Build id: 3.10.0-1160.6.1.el7.x86_64
0x96
15
__do_softirq
vmlinux 0xffffffff8c4a4b95
Build id: 3.10.0-1160.6.1.el7.x86_64
0xf5
16
run_ksoftirqd
vmlinux 0xffffffff8c4a4d58
Build id: 3.10.0-1160.6.1.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffff8c4ce674
Build id: 3.10.0-1160.6.1.el7.x86_64
0x144
18
? lg_double_unlock
vmlinux 0xffffffff8c4ce530
Build id: 3.10.0-1160.6.1.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffff8c4c5c21
Build id: 3.10.0-1160.6.1.el7.x86_64
0xd1
20
? insert_kthread_work
vmlinux 0xffffffff8c4c5b50
Build id: 3.10.0-1160.6.1.el7.x86_64
0x40
21
ret_from_fork_nospec_begin
vmlinux 0xffffffff8cb94df7
Build id: 3.10.0-1160.6.1.el7.x86_64
0x21
22
? insert_kthread_work
vmlinux 0xffffffff8c4c5b50
Build id: 3.10.0-1160.6.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:6]
IPv4: martian source 10.1.134.255 from 10.1.134.84, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b f9 81 08 00        .......PV.....
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00        .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00        .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00        .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00        .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00        .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.117, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 9b 40 70 08 00        .......PV.@p..
IPv4: martian source 10.1.134.255 from 10.1.134.183, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 be 34 26 08 00        .......PV.4&..
IPv4: martian source 10.1.134.255 from 10.1.134.183, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 be 34 26 08 00        .......PV.4&..
IPv4: martian source 10.1.134.255 from 10.1.134.183, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 50 56 be 34 26 08 00        .......PV.4&..
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm dns_resolver bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon pcspkr joydev sg parport_pc parport i2c_piix4 vmw_vmci xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper ata_piix syscopyarea ahci sysfillrect sysimgblt fb_sys_fops ttm libahci crct10dif_pclmul crct10dif_common crc32c_intel drm libata nfit libnvdimm serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: CyProtectDrv]
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G           OEL ------------   3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8e7cf2375280 ti: ffff8e7cf2f74000 task.ti: ffff8e7cf2f74000
RIP: 0010:[<ffffffff8cb89845>]  [<ffffffff8cb89845>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8e7cf2f77b08  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff8e7cf2f77ac8 RCX: ffffffff8d034f40
RDX: ffffb85e83130000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8e7cf2f77b08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8e7cd8bfb000 R11: ffff8e7cd83de000 R12: 0000000000000020
R13: ffff8e7f271ade00 R14: ffffffff8c8ead99 R15: ffff8e7cf2f77a80
FS:  0000000000000000(0000) GS:ffff8e839d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd0e3d23000 CR3: 00000003b2d70000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc03c7632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8c8ec0e0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8c8e39d0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8c8ed0fc>] scsi_request_fn+0x4ac/0x680
[<ffffffff8c753519>] __blk_run_queue+0x39/0x50
[<ffffffff8c753596>] blk_run_queue+0x26/0x40
[<ffffffff8c8eb658>] scsi_run_queue+0x258/0x2f0
[<ffffffff8c8ed310>] scsi_next_command+0x20/0x40
[<ffffffff8c8ed465>] scsi_end_request+0x135/0x1e0
[<ffffffff8c8ed6d8>] scsi_io_completion+0x168/0x720
[<ffffffff8c8e29cc>] scsi_finish_command+0xdc/0x140
[<ffffffff8c8ecc20>] scsi_softirq_done+0x130/0x160
[<ffffffff8c75d266>] blk_done_softirq+0x96/0xc0
[<ffffffff8c4a4b95>] __do_softirq+0xf5/0x280
[<ffffffff8c4a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff8c4ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8c4ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffff8c4c5c21>] kthread+0xd1/0xe0
[<ffffffff8c4c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8cb94df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8c4c5b50>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #53716
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc013a3be
Build id: 3.10.0-1127.18.2.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc032571e
Build id: 3.10.0-1127.18.2.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc016f2e0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff9c6e2950
Build id: 3.10.0-1127.18.2.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff9c6ec07c
Build id: 3.10.0-1127.18.2.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffff9c552769
Build id: 3.10.0-1127.18.2.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff9c5527e6
Build id: 3.10.0-1127.18.2.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff9c6ea5d8
Build id: 3.10.0-1127.18.2.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff9c6ec290
Build id: 3.10.0-1127.18.2.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff9c6ec3e5
Build id: 3.10.0-1127.18.2.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff9c6ec658
Build id: 3.10.0-1127.18.2.el7.x86_64
0x168
12
? queue_work_on
vmlinux 0xffffffff9c2bcadd
Build id: 3.10.0-1127.18.2.el7.x86_64
0x2d
13
scsi_finish_command
vmlinux 0xffffffff9c6e194c
Build id: 3.10.0-1127.18.2.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffff9c6ebba0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x130
15
blk_done_softirq
vmlinux 0xffffffff9c55c4b6
Build id: 3.10.0-1127.18.2.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffff9c2a5695
Build id: 3.10.0-1127.18.2.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffff9c2a5858
Build id: 3.10.0-1127.18.2.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff9c2cf0e4
Build id: 3.10.0-1127.18.2.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffff9c2cefa0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffff9c2c6691
Build id: 3.10.0-1127.18.2.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffff9c2c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffff9c992d37
Build id: 3.10.0-1127.18.2.el7.x86_64
0x21
23
? insert_kthread_work
vmlinux 0xffffffff9c2c65c0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ksoftirqd/7:45]
Modules linked in: xt_multiport xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nf_nat_ftp nf_conntrack_ftp ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp ppdev iosf_mbi crc32_pclmul vmw_balloon ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg pcspkr joydev i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c
sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi scsi_transport_spi ata_piix mptscsih drm crct10dif_pclmul crct10dif_common serio_raw libata crc32c_intel mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 7 PID: 45 Comm: ksoftirqd/7 Kdump: loaded Tainted: G             L ------------   3.10.0-1127.18.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013
task: ffff96c1367bc1c0 ti: ffff96c1367cc000 task.ti: ffff96c1367cc000
RIP: 0010:[<ffffffffc013a3be>]  [<ffffffffc013a3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff96c1367cfaa0  EFLAGS: 00000246
RAX: ffffb210c6a80000 RBX: ffff96c5f62d5200 RCX: 0000000000000025
RDX: ffff96cbedfa3e00 RSI: ffff96cc31200000 RDI: 000000000000000e
RBP: ffff96c1367cfab0 R08: 0000000000000003 R09: 0000000b83d2afff
R10: ffff96c60f1e3fd0 R11: ffffe49259382f00 R12: 0000000100000001
R13: ffff96cb4d494a80 R14: ffff96c1367cfa14 R15: ffffffff9c55b08b
FS:  0000000000000000(0000) GS:ffff96c60f1c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fafc26d8068 CR3: 0000000c68e5c000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc032571e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc016f2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9c6e2950>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9c6ec07c>] scsi_request_fn+0x4ac/0x680
[<ffffffff9c552769>] __blk_run_queue+0x39/0x50
[<ffffffff9c5527e6>] blk_run_queue+0x26/0x40
[<ffffffff9c6ea5d8>] scsi_run_queue+0x258/0x2f0
[<ffffffff9c6ec290>] scsi_next_command+0x20/0x40
[<ffffffff9c6ec3e5>] scsi_end_request+0x135/0x1e0
[<ffffffff9c6ec658>] scsi_io_completion+0x168/0x720
[<ffffffff9c2bcadd>] ? queue_work_on+0x2d/0x50
[<ffffffff9c6e194c>] scsi_finish_command+0xdc/0x140
[<ffffffff9c6ebba0>] scsi_softirq_done+0x130/0x160
[<ffffffff9c55c4b6>] blk_done_softirq+0x96/0xc0
[<ffffffff9c2a5695>] __do_softirq+0xf5/0x280
[<ffffffff9c2a5858>] run_ksoftirqd+0x38/0x50
[<ffffffff9c2cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9c2cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffff9c2c6691>] kthread+0xd1/0xe0
[<ffffffff9c2c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff9c992d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9c2c65c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 83 14 
Complete report #22894
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffa00773be
Build id: 3.10.0-514.10.2.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffa004f729
Build id: 3.10.0-514.10.2.el7.x86_64
0x249
3
mptspi_qcmd
mptspi 0xffffffffa01b12b0
Build id: 3.10.0-514.10.2.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff81451cba
Build id: 3.10.0-514.10.2.el7.x86_64
0xaa
5
scsi_request_fn
vmlinux 0xffffffff8145adc1
Build id: 3.10.0-514.10.2.el7.x86_64
0x501
6
__blk_run_queue
vmlinux 0xffffffff812eb253
Build id: 3.10.0-514.10.2.el7.x86_64
0x33
7
blk_run_queue
vmlinux 0xffffffff812eb2c6
Build id: 3.10.0-514.10.2.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff81459158
Build id: 3.10.0-514.10.2.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff8145b070
Build id: 3.10.0-514.10.2.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff8145b1c5
Build id: 3.10.0-514.10.2.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff8145b3e6
Build id: 3.10.0-514.10.2.el7.x86_64
0x116
12
? __netif_receive_skb
vmlinux 0xffffffff8156fd48
Build id: 3.10.0-514.10.2.el7.x86_64
0x18
13
scsi_finish_command
vmlinux 0xffffffff81450595
Build id: 3.10.0-514.10.2.el7.x86_64
0xd5
14
scsi_softirq_done
vmlinux 0xffffffff8145a892
Build id: 3.10.0-514.10.2.el7.x86_64
0x132
15
blk_done_softirq
vmlinux 0xffffffff812f5bb0
Build id: 3.10.0-514.10.2.el7.x86_64
0x90
16
__do_softirq
vmlinux 0xffffffff8108f2cf
Build id: 3.10.0-514.10.2.el7.x86_64
0xef
17
run_ksoftirqd
vmlinux 0xffffffff8108f498
Build id: 3.10.0-514.10.2.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff810b927f
Build id: 3.10.0-514.10.2.el7.x86_64
0x12f
19
? lg_double_unlock
vmlinux 0xffffffff810b9150
Build id: 3.10.0-514.10.2.el7.x86_64
0x90
20
kthread
vmlinux 0xffffffff810b06ff
Build id: 3.10.0-514.10.2.el7.x86_64
0xcf
21
? kthread_create_on_node
vmlinux 0xffffffff810b0630
Build id: 3.10.0-514.10.2.el7.x86_64
0x140
22
ret_from_fork
vmlinux 0xffffffff81696a58
Build id: 3.10.0-514.10.2.el7.x86_64
0x58
23
? kthread_create_on_node
vmlinux 0xffffffff810b0630
Build id: 3.10.0-514.10.2.el7.x86_64
0x140
NMI watchdog: BUG: soft lockup - CPU#13 stuck for 25s! [ksoftirqd/13:80]
Modules linked in: binfmt_misc intel_powerclamp coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr sg shpchp i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common mptspi drm ata_piix crc32c_intel serio_raw scsi_transport_spi mptscsih libata vmxnet3 mptbase i2c_core floppy fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 13 PID: 80 Comm: ksoftirqd/13 Tainted: G             L ------------   3.10.0-514.10.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff880fbbca1f60 ti: ffff880fbbcac000 task.ti: ffff880fbbcac000
RIP: 0010:[<ffffffffa00773be>]  [<ffffffffa00773be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff880fbbcafaa0  EFLAGS: 00000246
RAX: ffffc9000ca80000 RBX: ffff880fbbcafa70 RCX: 000000000000000f
RDX: ffff881435485400 RSI: ffff880f35674000 RDI: 000000000000000e
RBP: ffff880fbbcafab0 R08: 0000000000000003 R09: ffff8811b5c590d8
R10: ffff8810367379c0 R11: ffffea001b58fa00 R12: 0000000100000002
R13: ffff8801d7343600 R14: ffff880fbbcafa14 R15: ffffffff812f4c0b
FS:  0000000000000000(0000) GS:ffff8811b5d00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f25b89c1910 CR3: 00000000019ba000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880f35674d30 0000000000000048 ffff880fbbcafb58 ffffffffa004f729
ffff880f35674008 04000000bbcafb18 000000000000000f ffff8810367379c0
000000000000003c ffff880f35674188 000000600000000f ffff880a35202dd0
Call Trace:
[<ffffffffa004f729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa01b12b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff81451cba>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8145adc1>] scsi_request_fn+0x501/0x770
[<ffffffff812eb253>] __blk_run_queue+0x33/0x40
[<ffffffff812eb2c6>] blk_run_queue+0x26/0x40
[<ffffffff81459158>] scsi_run_queue+0x258/0x2f0
[<ffffffff8145b070>] scsi_next_command+0x20/0x40
[<ffffffff8145b1c5>] scsi_end_request+0x135/0x1e0
[<ffffffff8145b3e6>] scsi_io_completion+0x116/0x670
[<ffffffff8156fd48>] ? __netif_receive_skb+0x18/0x60
[<ffffffff81450595>] scsi_finish_command+0xd5/0x130
[<ffffffff8145a892>] scsi_softirq_done+0x132/0x160
[<ffffffff812f5bb0>] blk_done_softirq+0x90/0xc0
[<ffffffff8108f2cf>] __do_softirq+0xef/0x280
[<ffffffff8108f498>] run_ksoftirqd+0x38/0x50
[<ffffffff810b927f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b9150>] ? lg_double_unlock+0x90/0x90
[<ffffffff810b06ff>] kthread+0xcf/0xe0
[<ffffffff810b0630>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81696a58>] ret_from_fork+0x58/0x90
[<ffffffff810b0630>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 08 
Complete report #19496
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffa01b13be
Build id: 3.10.0-514.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffa0112729
Build id: 3.10.0-514.el7.x86_64
0x249
3
mptspi_qcmd
mptspi 0xffffffffa01d22b0
Build id: 3.10.0-514.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff814518da
Build id: 3.10.0-514.el7.x86_64
0xaa
5
scsi_request_fn
vmlinux 0xffffffff8145a9e1
Build id: 3.10.0-514.el7.x86_64
0x501
6
? __wake_up_common
vmlinux 0xffffffff810ba118
Build id: 3.10.0-514.el7.x86_64
0x58
7
__blk_run_queue
vmlinux 0xffffffff812eb053
Build id: 3.10.0-514.el7.x86_64
0x33
8
blk_run_queue
vmlinux 0xffffffff812eb0c6
Build id: 3.10.0-514.el7.x86_64
0x26
9
scsi_run_queue
vmlinux 0xffffffff81458d78
Build id: 3.10.0-514.el7.x86_64
0x258
10
scsi_next_command
vmlinux 0xffffffff8145ac90
Build id: 3.10.0-514.el7.x86_64
0x20
11
scsi_end_request
vmlinux 0xffffffff8145ade5
Build id: 3.10.0-514.el7.x86_64
0x135
12
scsi_io_completion
vmlinux 0xffffffff8145b006
Build id: 3.10.0-514.el7.x86_64
0x116
13
scsi_finish_command
vmlinux 0xffffffff814501b5
Build id: 3.10.0-514.el7.x86_64
0xd5
14
scsi_softirq_done
vmlinux 0xffffffff8145a4b2
Build id: 3.10.0-514.el7.x86_64
0x132
15
blk_done_softirq
vmlinux 0xffffffff812f59b0
Build id: 3.10.0-514.el7.x86_64
0x90
16
__do_softirq
vmlinux 0xffffffff8108f0ff
Build id: 3.10.0-514.el7.x86_64
0xef
17
run_ksoftirqd
vmlinux 0xffffffff8108f2c8
Build id: 3.10.0-514.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff810b90af
Build id: 3.10.0-514.el7.x86_64
0x12f
19
? lg_double_unlock
vmlinux 0xffffffff810b8f80
Build id: 3.10.0-514.el7.x86_64
0x90
20
kthread
vmlinux 0xffffffff810b052f
Build id: 3.10.0-514.el7.x86_64
0xcf
21
? kthread_create_on_node
vmlinux 0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
0x140
22
ret_from_fork
vmlinux 0xffffffff81696518
Build id: 3.10.0-514.el7.x86_64
0x58
23
? kthread_create_on_node
vmlinux 0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
0x140
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:3]
Modules linked in: tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event intel_powerclamp coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel snd_ens1371 lrw gf128mul glue_helper ablk_helper ppdev snd_rawmidi snd_ac97_codec cryptd ac97_bus snd_seq snd_seq_device snd_pcm snd_timer
snd soundcore vmw_balloon pcspkr sg nfit libnvdimm parport_pc parport vmw_vmci shpchp i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_piix vmwgfx e1000 mptspi scsi_transport_spi mptscsih mptbase drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm i2c_core libata fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Not tainted 3.10.0-514.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff88003bec9f60 ti: ffff88003bfd8000 task.ti: ffff88003bfd8000
RIP: 0010:[<ffffffffa01b13be>]  [<ffffffffa01b13be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88003bfdbaa0  EFLAGS: 00000246
RAX: ffffc900002c0000 RBX: ffff880037a8eec0 RCX: 000000000000000c
RDX: ffff880035a30600 RSI: ffff880035a38000 RDI: 000000000000000e
RBP: ffff88003bfdbab0 R08: 0000000000000003 R09: 000000001cc31fff
R10: ffff8800001e2bc0 R11: ffffea0000d65600 R12: 0000000100000037
R13: ffff88000fff8c00 R14: ffff88003bfdba14 R15: ffffffff812f4a0b
FS:  0000000000000000(0000) GS:ffff88003c600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f024b379000 CR3: 0000000033d75000 CR4: 00000000003407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880035a38d30 0000000000000030 ffff88003bfdbb58 ffffffffa0112729
ffff880035a38008 04000000783f4b4a 000000000000000c ffff8800001e2bc0
0000000000000030 ffff880035a38188 000000600000000c ffff88000000000b
Call Trace:
[<ffffffffa0112729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa01d22b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff814518da>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8145a9e1>] scsi_request_fn+0x501/0x770
[<ffffffff810ba118>] ? __wake_up_common+0x58/0x90
[<ffffffff812eb053>] __blk_run_queue+0x33/0x40
[<ffffffff812eb0c6>] blk_run_queue+0x26/0x40
[<ffffffff81458d78>] scsi_run_queue+0x258/0x2f0
[<ffffffff8145ac90>] scsi_next_command+0x20/0x40
[<ffffffff8145ade5>] scsi_end_request+0x135/0x1e0
[<ffffffff8145b006>] scsi_io_completion+0x116/0x670
[<ffffffff814501b5>] scsi_finish_command+0xd5/0x130
[<ffffffff8145a4b2>] scsi_softirq_done+0x132/0x160
[<ffffffff812f59b0>] blk_done_softirq+0x90/0xc0
[<ffffffff8108f0ff>] __do_softirq+0xef/0x280
[<ffffffff8108f2c8>] run_ksoftirqd+0x38/0x50
[<ffffffff810b90af>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f80>] ? lg_double_unlock+0x90/0x90
[<ffffffff810b052f>] kthread+0xcf/0xe0
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81696518>] ret_from_fork+0x58/0x90
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 f3 1b 
Complete report #16208
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffa011d3be
Build id: 3.10.0-327.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffa008f729
Build id: 3.10.0-327.el7.x86_64
0x249
3
mptspi_qcmd
mptspi 0xffffffffa00462b0
Build id: 3.10.0-327.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff81417b1a
Build id: 3.10.0-327.el7.x86_64
0xaa
5
scsi_request_fn
vmlinux 0xffffffff81420aa1
Build id: 3.10.0-327.el7.x86_64
0x501
6
__blk_run_queue
vmlinux 0xffffffff812c73e3
Build id: 3.10.0-327.el7.x86_64
0x33
7
blk_run_queue
vmlinux 0xffffffff812c7456
Build id: 3.10.0-327.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff8141eec8
Build id: 3.10.0-327.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff81420d50
Build id: 3.10.0-327.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff81420ec5
Build id: 3.10.0-327.el7.x86_64
0x155
11
scsi_io_completion
vmlinux 0xffffffff814210a3
Build id: 3.10.0-327.el7.x86_64
0x103
12
scsi_finish_command
vmlinux 0xffffffff814163e5
Build id: 3.10.0-327.el7.x86_64
0xd5
13
scsi_softirq_done
vmlinux 0xffffffff8142057a
Build id: 3.10.0-327.el7.x86_64
0x12a
14
blk_done_softirq
vmlinux 0xffffffff812d16a0
Build id: 3.10.0-327.el7.x86_64
0x90
15
__do_softirq
vmlinux 0xffffffff81084b0f
Build id: 3.10.0-327.el7.x86_64
0xef
16
run_ksoftirqd
vmlinux 0xffffffff81084cd0
Build id: 3.10.0-327.el7.x86_64
0x30
17
smpboot_thread_fn
vmlinux 0xffffffff810ae52f
Build id: 3.10.0-327.el7.x86_64
0xff
18
? schedule
vmlinux 0xffffffff8163a909
Build id: 3.10.0-327.el7.x86_64
0x29
19
? lg_double_unlock
vmlinux 0xffffffff810ae430
Build id: 3.10.0-327.el7.x86_64
0x90
20
kthread
vmlinux 0xffffffff810a5aef
Build id: 3.10.0-327.el7.x86_64
0xcf
21
? kthread_create_on_node
vmlinux 0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
0x140
22
ret_from_fork
vmlinux 0xffffffff81645858
Build id: 3.10.0-327.el7.x86_64
0x58
23
? kthread_create_on_node
vmlinux 0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
0x140
BUG: soft lockup - CPU#1 stuck for 24s! [ksoftirqd/1:21]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_lsm_pinned_10703(E) crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg ppdev pcspkr vmw_balloon vmxnet3 parport_pc i2c_piix4 parport vmw_vmci shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel serio_raw e1000 vmwgfx mptspi drm_kms_helper scsi_transport_spi mptscsih
ttm mptbase ata_piix drm libata i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 21 Comm: ksoftirqd/1 Tainted: P            E  ------------   3.10.0-327.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff880429610b80 ti: ffff88042961c000 task.ti: ffff88042961c000
RIP: 0010:[<ffffffffa011d3be>]  [<ffffffffa011d3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88042961fa80  EFLAGS: 00000246
RAX: ffffc90009b00000 RBX: ffff88042961fa50 RCX: 0000000000000000
RDX: ffff880423c64200 RSI: ffff880035d81000 RDI: 000000000000000e
RBP: ffff88042961fa90 R08: 0000000000000003 R09: ffff880035d860d8
R10: ffff8801aad4d340 R11: ffffea0006ab5300 R12: 0000000100000080
R13: ffff88010f458300 R14: ffff88042961f9f4 R15: ffffffff812d06bb
FS:  0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc0b2043000 CR3: 000000018f440000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880035d81d30 0000000000000048 ffff88042961fb38 ffffffffa008f729
ffff880035d81008 0400000081631ada 0000000000000000 ffff8801aad4d340
0000000000000000 ffff880035d81188 0000006000000000 ffff880400000007
Call Trace:
[<ffffffffa008f729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa00462b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff81417b1a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff81420aa1>] scsi_request_fn+0x501/0x770
[<ffffffff812c73e3>] __blk_run_queue+0x33/0x40
[<ffffffff812c7456>] blk_run_queue+0x26/0x40
[<ffffffff8141eec8>] scsi_run_queue+0x258/0x2f0
[<ffffffff81420d50>] scsi_next_command+0x20/0x40
[<ffffffff81420ec5>] scsi_end_request+0x155/0x1d0
[<ffffffff814210a3>] scsi_io_completion+0x103/0x600
[<ffffffff814163e5>] scsi_finish_command+0xd5/0x130
[<ffffffff8142057a>] scsi_softirq_done+0x12a/0x150
[<ffffffff812d16a0>] blk_done_softirq+0x90/0xc0
[<ffffffff81084b0f>] __do_softirq+0xef/0x280
[<ffffffff81084cd0>] run_ksoftirqd+0x30/0x50
[<ffffffff810ae52f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163a909>] ? schedule+0x29/0x70
[<ffffffff810ae430>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5aef>] kthread+0xcf/0xe0
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81645858>] ret_from_fork+0x58/0x90
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 12 
Complete report #12175
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc04793be
Build id: 3.10.0-957.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc049b71e
Build id: 3.10.0-957.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc04942e0
Build id: 3.10.0-957.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff908d0960
Build id: 3.10.0-957.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff908d9efc
Build id: 3.10.0-957.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffff90742ff9
Build id: 3.10.0-957.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffff90743076
Build id: 3.10.0-957.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff908d84a8
Build id: 3.10.0-957.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff908da0f0
Build id: 3.10.0-957.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff908da245
Build id: 3.10.0-957.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff908da4b8
Build id: 3.10.0-957.el7.x86_64
0x168
12
? __netif_receive_skb
vmlinux 0xffffffff90a397f8
Build id: 3.10.0-957.el7.x86_64
0x18
13
scsi_finish_command
vmlinux 0xffffffff908cf96c
Build id: 3.10.0-957.el7.x86_64
0xdc
14
scsi_softirq_done
vmlinux 0xffffffff908d9a02
Build id: 3.10.0-957.el7.x86_64
0x132
15
blk_done_softirq
vmlinux 0xffffffff9074cce6
Build id: 3.10.0-957.el7.x86_64
0x96
16
__do_softirq
vmlinux 0xffffffff904a0f05
Build id: 3.10.0-957.el7.x86_64
0xf5
17
run_ksoftirqd
vmlinux 0xffffffff904a10c8
Build id: 3.10.0-957.el7.x86_64
0x38
18
smpboot_thread_fn
vmlinux 0xffffffff904ca5c4
Build id: 3.10.0-957.el7.x86_64
0x144
19
? lg_double_unlock
vmlinux 0xffffffff904ca480
Build id: 3.10.0-957.el7.x86_64
0x40
20
kthread
vmlinux 0xffffffff904c1c31
Build id: 3.10.0-957.el7.x86_64
0xd1
21
? insert_kthread_work
vmlinux 0xffffffff904c1b60
Build id: 3.10.0-957.el7.x86_64
0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffff90b74c1d
Build id: 3.10.0-957.el7.x86_64
0x7
23
? insert_kthread_work
vmlinux 0xffffffff904c1b60
Build id: 3.10.0-957.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [ksoftirqd/6:40]
Modules linked in: binfmt_misc veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_nat_ipv4 xt_addrtype nf_nat br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper vmw_balloon ablk_helper cryptd joydev pcspkr sg nfit parport_pc vmw_vmci parport libnvdimm i2c_piix4 ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul drm crct10dif_common crc32c_intel serio_raw drm_panel_orientation_quirks mptspi vmxnet3 scsi_transport_spi
mptscsih mptbase floppy ata_piix libata dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 40 Comm: ksoftirqd/6 Kdump: loaded Tainted: G        W    L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9af3f4e51040 ti: ffff9af3f4e5c000 task.ti: ffff9af3f4e5c000
RIP: 0010:[<ffffffffc04793be>]  [<ffffffffc04793be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9af3f4e5faa0  EFLAGS: 00000246
RAX: ffffb0f706a00000 RBX: ffff9af35921ffc0 RCX: 0000000000000018
RDX: ffff9afb5dfe7000 RSI: ffff9afb5dc7e000 RDI: 000000000000000e
RBP: ffff9af3f4e5fab0 R08: 0000000000000003 R09: 000000021d014fff
R10: ffff9afb5dcda0d8 R11: ffff9af775e7ebc0 R12: 000000010000007f
R13: ffff9aec546bbb00 R14: ffff9af3f4e5fa14 R15: ffffffff9074b83b
FS:  0000000000000000(0000) GS:ffff9afb7fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff1ba8be000 CR3: 0000000315410000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc049b71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04942e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff908d0960>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff908d9efc>] scsi_request_fn+0x4cc/0x680
[<ffffffff90742ff9>] __blk_run_queue+0x39/0x50
[<ffffffff90743076>] blk_run_queue+0x26/0x40
[<ffffffff908d84a8>] scsi_run_queue+0x258/0x2f0
[<ffffffff908da0f0>] scsi_next_command+0x20/0x40
[<ffffffff908da245>] scsi_end_request+0x135/0x1e0
[<ffffffff908da4b8>] scsi_io_completion+0x168/0x6a0
[<ffffffff90a397f8>] ? __netif_receive_skb+0x18/0x60
[<ffffffff908cf96c>] scsi_finish_command+0xdc/0x140
[<ffffffff908d9a02>] scsi_softirq_done+0x132/0x160
[<ffffffff9074cce6>] blk_done_softirq+0x96/0xc0
[<ffffffff904a0f05>] __do_softirq+0xf5/0x280
[<ffffffff904a10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff904ca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff904ca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff904c1c31>] kthread+0xd1/0xe0
[<ffffffff904c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff90b74c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff904c1b60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 73 48 
Complete report #7798
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffa00b73be
Build id: 3.10.0-327.28.3.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffa00d2729
Build id: 3.10.0-327.28.3.el7.x86_64
0x249
3
mptspi_qcmd
mptspi 0xffffffffa004d2b0
Build id: 3.10.0-327.28.3.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8141845a
Build id: 3.10.0-327.28.3.el7.x86_64
0xaa
5
scsi_request_fn
vmlinux 0xffffffff814213e1
Build id: 3.10.0-327.28.3.el7.x86_64
0x501
6
__blk_run_queue
vmlinux 0xffffffff812c7c63
Build id: 3.10.0-327.28.3.el7.x86_64
0x33
7
blk_run_queue
vmlinux 0xffffffff812c7cd6
Build id: 3.10.0-327.28.3.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff8141f808
Build id: 3.10.0-327.28.3.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff81421690
Build id: 3.10.0-327.28.3.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff81421805
Build id: 3.10.0-327.28.3.el7.x86_64
0x155
11
scsi_io_completion
vmlinux 0xffffffff814219e3
Build id: 3.10.0-327.28.3.el7.x86_64
0x103
12
? list_del
vmlinux 0xffffffff8130cd4d
Build id: 3.10.0-327.28.3.el7.x86_64
0xd
13
scsi_finish_command
vmlinux 0xffffffff81416d25
Build id: 3.10.0-327.28.3.el7.x86_64
0xd5
14
scsi_softirq_done
vmlinux 0xffffffff81420eba
Build id: 3.10.0-327.28.3.el7.x86_64
0x12a
15
blk_done_softirq
vmlinux 0xffffffff812d1f80
Build id: 3.10.0-327.28.3.el7.x86_64
0x90
16
__do_softirq
vmlinux 0xffffffff81084b4f
Build id: 3.10.0-327.28.3.el7.x86_64
0xef
17
run_ksoftirqd
vmlinux 0xffffffff81084d10
Build id: 3.10.0-327.28.3.el7.x86_64
0x30
18
smpboot_thread_fn
vmlinux 0xffffffff810ae58f
Build id: 3.10.0-327.28.3.el7.x86_64
0xff
19
? schedule
vmlinux 0xffffffff8163bb59
Build id: 3.10.0-327.28.3.el7.x86_64
0x29
20
? lg_double_unlock
vmlinux 0xffffffff810ae490
Build id: 3.10.0-327.28.3.el7.x86_64
0x90
21
kthread
vmlinux 0xffffffff810a5b2f
Build id: 3.10.0-327.28.3.el7.x86_64
0xcf
22
? kthread_create_on_node
vmlinux 0xffffffff810a5a60
Build id: 3.10.0-327.28.3.el7.x86_64
0x140
23
ret_from_fork
vmlinux 0xffffffff81646a98
Build id: 3.10.0-327.28.3.el7.x86_64
0x58
24
? kthread_create_on_node
vmlinux 0xffffffff810a5a60
Build id: 3.10.0-327.28.3.el7.x86_64
0x140
BUG: soft lockup - CPU#1 stuck for 57s! [tuned:2323]
INFO: rcu_sched detected stalls on CPUs/tasks: {} (detected by 0, t=75895 jiffies, g=14357339, c=14357338, q=0)
INFO: Stall ended before state dump start
BUG: soft lockup - CPU#0 stuck for 57s! [ksoftirqd/0:3]
Modules linked in: ppdev coretemp vmw_balloon pcspkr sg parport_pc i2c_piix4 parport shpchp vmw_vmci ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi crc32c_intel serio_raw vmwgfx mptspi drm_kms_helper ttm scsi_transport_spi drm mptscsih vmxnet3 mptbase ata_piix i2c_core libata floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Tainted: G             L ------------   3.10.0-327.28.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff880139b11700 ti: ffff880139b90000 task.ti: ffff880139b90000
RIP: 0010:[<ffffffffa00b73be>]  [<ffffffffa00b73be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff880139b93a80  EFLAGS: 00000246
RAX: ffffc900087c0000 RBX: ffff880139b93a50 RCX: 0000000000000000
RDX: ffff880036a50c00 RSI: ffff880135c5a000 RDI: 000000000000000e
RBP: ffff880139b93a90 R08: 0000000000000003 R09: ffff8800360150d8
R10: ffff88009bb28540 R11: ffffea00026eca00 R12: 0000000100000002
R13: ffff8800ae8c8300 R14: ffff880139b939f4 R15: ffffffff812d0f9b
FS:  0000000000000000(0000) GS:ffff88013fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007efee14e7000 CR3: 00000000566cd000 CR4: 00000000000007f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880135c5ad30 0000000000000048 ffff880139b93b38 ffffffffa00d2729
ffff880135c5a008 04000000d96c3de6 0000000000000000 ffff88009bb28540
0000000000000000 ffff880135c5a188 0000006000000000 ffff880036182830
Call Trace:
[<ffffffffa00d2729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa004d2b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff8141845a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff814213e1>] scsi_request_fn+0x501/0x770
[<ffffffff812c7c63>] __blk_run_queue+0x33/0x40
[<ffffffff812c7cd6>] blk_run_queue+0x26/0x40
[<ffffffff8141f808>] scsi_run_queue+0x258/0x2f0
[<ffffffff81421690>] scsi_next_command+0x20/0x40
[<ffffffff81421805>] scsi_end_request+0x155/0x1d0
[<ffffffff814219e3>] scsi_io_completion+0x103/0x600
[<ffffffff8130cd4d>] ? list_del+0xd/0x30
[<ffffffff81416d25>] scsi_finish_command+0xd5/0x130
[<ffffffff81420eba>] scsi_softirq_done+0x12a/0x150
[<ffffffff812d1f80>] blk_done_softirq+0x90/0xc0
[<ffffffff81084b4f>] __do_softirq+0xef/0x280
[<ffffffff81084d10>] run_ksoftirqd+0x30/0x50
[<ffffffff810ae58f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163bb59>] ? schedule+0x29/0x70
[<ffffffff810ae490>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5b2f>] kthread+0xcf/0xe0
[<ffffffff810a5a60>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81646a98>] ret_from_fork+0x58/0x90
[<ffffffff810a5a60>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 0c 
Complete report #6584
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01173be
Build id: 3.10.0-1127.19.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc013271e
Build id: 3.10.0-1127.19.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc010b2e0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa2ce2990
Build id: 3.10.0-1127.19.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffa2cec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffa2b527a9
Build id: 3.10.0-1127.19.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffa2b52826
Build id: 3.10.0-1127.19.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffa2cea618
Build id: 3.10.0-1127.19.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffa2cec2d0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffa2cec425
Build id: 3.10.0-1127.19.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffa2cec698
Build id: 3.10.0-1127.19.1.el7.x86_64
0x168
12
scsi_finish_command
vmlinux 0xffffffffa2ce198c
Build id: 3.10.0-1127.19.1.el7.x86_64
0xdc
13
scsi_softirq_done
vmlinux 0xffffffffa2cebbe0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x130
14
blk_done_softirq
vmlinux 0xffffffffa2b5c4f6
Build id: 3.10.0-1127.19.1.el7.x86_64
0x96
15
__do_softirq
vmlinux 0xffffffffa28a5695
Build id: 3.10.0-1127.19.1.el7.x86_64
0xf5
16
run_ksoftirqd
vmlinux 0xffffffffa28a5858
Build id: 3.10.0-1127.19.1.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffffa28cf0e4
Build id: 3.10.0-1127.19.1.el7.x86_64
0x144
18
? lg_double_unlock
vmlinux 0xffffffffa28cefa0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffffa28c6691
Build id: 3.10.0-1127.19.1.el7.x86_64
0xd1
20
? insert_kthread_work
vmlinux 0xffffffffa28c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
21
ret_from_fork_nospec_begin
vmlinux 0xffffffffa2f92d37
Build id: 3.10.0-1127.19.1.el7.x86_64
0x21
22
? insert_kthread_work
vmlinux 0xffffffffa28c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 43s! [ksoftirqd/1:14]
Modules linked in: vmfsrtx_dev(POE) vmfsrtx_dev_helper(POE) vmfsrtx_export(POE) vmfsrtx_bridge(POE) vmfsrtx_bridge_helper(OE) fuse tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd pcspkr joydev sg parport_pc parport vmw_vmci i2c_piix4 auth_rpcgss sunrpc
ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ttm crc32c_intel ata_piix drm serio_raw libata mptspi scsi_transport_spi mptscsih mptbase vmxnet3 nfit libnvdimm drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: vmfsrtx_dev_helper]
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: P           OEL ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff932278ed20e0 ti: ffff932278ee4000 task.ti: ffff932278ee4000
RIP: 0010:[<ffffffffc01173be>]  [<ffffffffc01173be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff932278ee7aa0  EFLAGS: 00000246
RAX: ffffafb2807c0000 RBX: ffff9321f9d83150 RCX: 0000000000000006
RDX: ffff932175efe800 RSI: ffff932175d53000 RDI: 000000000000000e
RBP: ffff932278ee7ab0 R08: 0000000000000002 R09: 000000011d5bbfff
R10: ffff93227ae63c00 R11: ffffec8044de9700 R12: 00000000005c2ed4
R13: 0000000078ee7a58 R14: ffffffffa28e5fca R15: ffff932278ee7a40
FS:  0000000000000000(0000) GS:ffff93227ae40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fdc51504000 CR3: 0000000135e34000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc013271e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc010b2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa2ce2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa2cec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffffa2b527a9>] __blk_run_queue+0x39/0x50
[<ffffffffa2b52826>] blk_run_queue+0x26/0x40
[<ffffffffa2cea618>] scsi_run_queue+0x258/0x2f0
[<ffffffffa2cec2d0>] scsi_next_command+0x20/0x40
[<ffffffffa2cec425>] scsi_end_request+0x135/0x1e0
[<ffffffffa2cec698>] scsi_io_completion+0x168/0x720
[<ffffffffa2ce198c>] scsi_finish_command+0xdc/0x140
[<ffffffffa2cebbe0>] scsi_softirq_done+0x130/0x160
[<ffffffffa2b5c4f6>] blk_done_softirq+0x96/0xc0
[<ffffffffa28a5695>] __do_softirq+0xf5/0x280
[<ffffffffa28a5858>] run_ksoftirqd+0x38/0x50
[<ffffffffa28cf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa28cefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa28c6691>] kthread+0xd1/0xe0
[<ffffffffa28c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2f92d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa28c65c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 12 
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [CRTC:37:crtc-0] flip_done timed out
Complete report #1812
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03f63be
Build id: 3.10.0-862.2.3.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc045071e
Build id: 3.10.0-862.2.3.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc04112e0
Build id: 3.10.0-862.2.3.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffadc9aff0
Build id: 3.10.0-862.2.3.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffadca455c
Build id: 3.10.0-862.2.3.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffffadb18a79
Build id: 3.10.0-862.2.3.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffadb18af6
Build id: 3.10.0-862.2.3.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffadca2af8
Build id: 3.10.0-862.2.3.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffadca4750
Build id: 3.10.0-862.2.3.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffadca48a5
Build id: 3.10.0-862.2.3.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffadca4b18
Build id: 3.10.0-862.2.3.el7.x86_64
0x168
12
scsi_finish_command
vmlinux 0xffffffffadc99ffc
Build id: 3.10.0-862.2.3.el7.x86_64
0xdc
13
scsi_softirq_done
vmlinux 0xffffffffadca4062
Build id: 3.10.0-862.2.3.el7.x86_64
0x132
14
blk_done_softirq
vmlinux 0xffffffffadb22736
Build id: 3.10.0-862.2.3.el7.x86_64
0x96
15
__do_softirq
vmlinux 0xffffffffad89a945
Build id: 3.10.0-862.2.3.el7.x86_64
0xf5
16
run_ksoftirqd
vmlinux 0xffffffffad89ab08
Build id: 3.10.0-862.2.3.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffffad8c35e4
Build id: 3.10.0-862.2.3.el7.x86_64
0x144
18
? lg_double_unlock
vmlinux 0xffffffffad8c34a0
Build id: 3.10.0-862.2.3.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffffad8bae31
Build id: 3.10.0-862.2.3.el7.x86_64
0xd1
20
? insert_kthread_work
vmlinux 0xffffffffad8bad60
Build id: 3.10.0-862.2.3.el7.x86_64
0x40
21
ret_from_fork_nospec_begin
vmlinux 0xffffffffadf1f5f7
Build id: 3.10.0-862.2.3.el7.x86_64
0x21
22
? insert_kthread_work
vmlinux 0xffffffffad8bad60
Build id: 3.10.0-862.2.3.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 28s! [ksoftirqd/3:24]
Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ip_set nfnetlink ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_security iptable_raw tcp_lp fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev vmw_balloon pcspkr sg nfit libnvdimm parport_pc parport shpchp vmw_vmci i2c_piix4 ip_tables xfs libcrc32c
sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix mptspi scsi_transport_spi mptscsih libata drm crct10dif_pclmul crct10dif_common e1000 crc32c_intel mptbase serio_raw i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G        W    L ------------   3.10.0-862.2.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9bc4bd7c1fa0 ti: ffff9bc4bd7d0000 task.ti: ffff9bc4bd7d0000
RIP: 0010:[<ffffffffc03f63be>]  [<ffffffffc03f63be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9bc4bd7d3aa0  EFLAGS: 00000246
RAX: ffffad76c0e80000 RBX: ffff9bc4f3765800 RCX: 0000000000000047
RDX: ffff9bc4f45cfe00 RSI: ffff9bc4f5284000 RDI: 000000000000000e
RBP: ffff9bc4bd7d3ab0 R08: 0000000000000002 R09: 00000001b44345ff
R10: ffff9bc4f52850d8 R11: ffff9bc4f22c8540 R12: 0000000100000001
R13: ffff9bc4ed37ad80 R14: ffff9bc4bd7d3a14 R15: ffffffffadb2126b
FS:  0000000000000000(0000) GS:ffff9bc4ffd80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fdd856e3080 CR3: 000000013d80e000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc045071e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04112e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffadc9aff0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffadca455c>] scsi_request_fn+0x4cc/0x680
[<ffffffffadb18a79>] __blk_run_queue+0x39/0x50
[<ffffffffadb18af6>] blk_run_queue+0x26/0x40
[<ffffffffadca2af8>] scsi_run_queue+0x258/0x2f0
[<ffffffffadca4750>] scsi_next_command+0x20/0x40
[<ffffffffadca48a5>] scsi_end_request+0x135/0x1e0
[<ffffffffadca4b18>] scsi_io_completion+0x168/0x6a0
[<ffffffffadc99ffc>] scsi_finish_command+0xdc/0x140
[<ffffffffadca4062>] scsi_softirq_done+0x132/0x160
[<ffffffffadb22736>] blk_done_softirq+0x96/0xc0
[<ffffffffad89a945>] __do_softirq+0xf5/0x280
[<ffffffffad89ab08>] run_ksoftirqd+0x38/0x50
[<ffffffffad8c35e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffad8c34a0>] ? lg_double_unlock+0x40/0x40
[<ffffffffad8bae31>] kthread+0xd1/0xe0
[<ffffffffad8bad60>] ? insert_kthread_work+0x40/0x40
[<ffffffffadf1f5f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffad8bad60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 43 40 
Complete report #1024
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffb1d83815
Build id: 3.10.0-1062.18.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0301632
Build id: 3.10.0-1062.18.1.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffffb1ae72d0
Build id: 3.10.0-1062.18.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffffb1adecd0
Build id: 3.10.0-1062.18.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb1ae833c
Build id: 3.10.0-1062.18.1.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffffb194f289
Build id: 3.10.0-1062.18.1.el7.x86_64
0x39
7
blk_run_queue
vmlinux 0xffffffffb194f306
Build id: 3.10.0-1062.18.1.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffffb1ae68e8
Build id: 3.10.0-1062.18.1.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffffb1ae8530
Build id: 3.10.0-1062.18.1.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffffb1ae8685
Build id: 3.10.0-1062.18.1.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffffb1ae88f8
Build id: 3.10.0-1062.18.1.el7.x86_64
0x168
12
scsi_finish_command
vmlinux 0xffffffffb1addcdc
Build id: 3.10.0-1062.18.1.el7.x86_64
0xdc
13
scsi_softirq_done
vmlinux 0xffffffffb1ae7e42
Build id: 3.10.0-1062.18.1.el7.x86_64
0x132
14
blk_done_softirq
vmlinux 0xffffffffb1958f96
Build id: 3.10.0-1062.18.1.el7.x86_64
0x96
15
__do_softirq
vmlinux 0xffffffffb16a5435
Build id: 3.10.0-1062.18.1.el7.x86_64
0xf5
16
run_ksoftirqd
vmlinux 0xffffffffb16a55f8
Build id: 3.10.0-1062.18.1.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffffb16ced54
Build id: 3.10.0-1062.18.1.el7.x86_64
0x144
18
? lg_double_unlock
vmlinux 0xffffffffb16cec10
Build id: 3.10.0-1062.18.1.el7.x86_64
0x40
19
kthread
vmlinux 0xffffffffb16c6321
Build id: 3.10.0-1062.18.1.el7.x86_64
0xd1
20
? insert_kthread_work
vmlinux 0xffffffffb16c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
0x40
21
ret_from_fork_nospec_begin
vmlinux 0xffffffffb1d8dd37
Build id: 3.10.0-1062.18.1.el7.x86_64
0x21
22
? insert_kthread_work
vmlinux 0xffffffffb16c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ksoftirqd/1:14]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 sb_edac ppdev iosf_mbi nfit crc32_pclmul ghash_clmulni_intel vmw_balloon libnvdimm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg joydev pcspkr parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm libahci crct10dif_pclmul crct10dif_common crc32c_intel ata_piix serio_raw libata
vmxnet3 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G             L ------------   3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/11/2017
task: ffff8cf27c418000 ti: ffff8cf27c414000 task.ti: ffff8cf27c414000
RIP: 0010:[<ffffffffb1d83815>]  [<ffffffffb1d83815>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8cf27c417b08  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff8cf27c417ac8 RCX: ffffffffb2234800
RDX: ffffb1a500cc0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8cf27c417b08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8cf32fc1d8d8 R11: ffff8cf330575800 R12: 0000000000000020
R13: ffff8cf254595080 R14: ffffffffb1ae6029 R15: ffff8cf27c417a80
FS:  0000000000000000(0000) GS:ffff8cf335640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6936097000 CR3: 00000000014b2000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc0301632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffb1ae72d0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffb1adecd0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb1ae833c>] scsi_request_fn+0x4cc/0x680
[<ffffffffb194f289>] __blk_run_queue+0x39/0x50
[<ffffffffb194f306>] blk_run_queue+0x26/0x40
[<ffffffffb1ae68e8>] scsi_run_queue+0x258/0x2f0
[<ffffffffb1ae8530>] scsi_next_command+0x20/0x40
[<ffffffffb1ae8685>] scsi_end_request+0x135/0x1e0
[<ffffffffb1ae88f8>] scsi_io_completion+0x168/0x6a0
[<ffffffffb1addcdc>] scsi_finish_command+0xdc/0x140
[<ffffffffb1ae7e42>] scsi_softirq_done+0x132/0x160
[<ffffffffb1958f96>] blk_done_softirq+0x96/0xc0
[<ffffffffb16a5435>] __do_softirq+0xf5/0x280
[<ffffffffb16a55f8>] run_ksoftirqd+0x38/0x50
[<ffffffffb16ced54>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb16cec10>] ? lg_double_unlock+0x40/0x40
[<ffffffffb16c6321>] kthread+0xd1/0xe0
[<ffffffffb16c6250>] ? insert_kthread_work+0x40/0x40
[<ffffffffb1d8dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb16c6250>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #161
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffa00773be
Build id: 3.10.0-514.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffa0092729
Build id: 3.10.0-514.el7.x86_64
0x249
3
mptspi_qcmd
mptspi 0xffffffffa01572b0
Build id: 3.10.0-514.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff814518da
Build id: 3.10.0-514.el7.x86_64
0xaa
5
scsi_request_fn
vmlinux 0xffffffff8145a9e1
Build id: 3.10.0-514.el7.x86_64
0x501
6
__blk_run_queue
vmlinux 0xffffffff812eb053
Build id: 3.10.0-514.el7.x86_64
0x33
7
blk_run_queue
vmlinux 0xffffffff812eb0c6
Build id: 3.10.0-514.el7.x86_64
0x26
8
scsi_run_queue
vmlinux 0xffffffff81458d78
Build id: 3.10.0-514.el7.x86_64
0x258
9
scsi_next_command
vmlinux 0xffffffff8145ac90
Build id: 3.10.0-514.el7.x86_64
0x20
10
scsi_end_request
vmlinux 0xffffffff8145ade5
Build id: 3.10.0-514.el7.x86_64
0x135
11
scsi_io_completion
vmlinux 0xffffffff8145b006
Build id: 3.10.0-514.el7.x86_64
0x116
12
scsi_finish_command
vmlinux 0xffffffff814501b5
Build id: 3.10.0-514.el7.x86_64
0xd5
13
scsi_softirq_done
vmlinux 0xffffffff8145a4b2
Build id: 3.10.0-514.el7.x86_64
0x132
14
blk_done_softirq
vmlinux 0xffffffff812f59b0
Build id: 3.10.0-514.el7.x86_64
0x90
15
__do_softirq
vmlinux 0xffffffff8108f0ff
Build id: 3.10.0-514.el7.x86_64
0xef
16
run_ksoftirqd
vmlinux 0xffffffff8108f2c8
Build id: 3.10.0-514.el7.x86_64
0x38
17
smpboot_thread_fn
vmlinux 0xffffffff810b90af
Build id: 3.10.0-514.el7.x86_64
0x12f
18
? lg_double_unlock
vmlinux 0xffffffff810b8f80
Build id: 3.10.0-514.el7.x86_64
0x90
19
kthread
vmlinux 0xffffffff810b052f
Build id: 3.10.0-514.el7.x86_64
0xcf
20
? kthread_create_on_node
vmlinux 0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
0x140
21
ret_from_fork
vmlinux 0xffffffff81696518
Build id: 3.10.0-514.el7.x86_64
0x58
22
? kthread_create_on_node
vmlinux 0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
0x140
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ksoftirqd/2:18]
Modules linked in: talpa_vfshook(OE) talpa_pedconnector(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) talpa_pedevice(OE) binfmt_misc tcp_diag inet_diag ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock intel_powerclamp coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon sg pcspkr
parport_pc shpchp parport vmw_vmci i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt ata_piix fb_sys_fops ttm libata crct10dif_pclmul mptspi crct10dif_common drm crc32c_intel serio_raw scsi_transport_spi vmxnet3 mptscsih mptbase i2c_core floppy fjes dm_mirror dm_region_hash dm_log dm_mod [last unloaded: talpa_syscallhook]
CPU: 2 PID: 18 Comm: ksoftirqd/2 Tainted: G           OEL ------------   3.10.0-514.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff880182278000 ti: ffff880182280000 task.ti: ffff880182280000
RIP: 0010:[<ffffffffa00773be>]  [<ffffffffa00773be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff880182283aa0  EFLAGS: 00000246
RAX: ffffc90001c00000 RBX: ffff880182283a70 RCX: 000000000000000d
RDX: ffff88042b8bde00 RSI: ffff88042c125000 RDI: 000000000000000e
RBP: ffff880182283ab0 R08: 0000000000000003 R09: ffff88042bbe30d8
R10: ffff88042a675340 R11: ffffea0010a99d00 R12: 0000000100000080
R13: ffff88040f6e0a80 R14: ffff880182283a14 R15: ffffffff812f4a0b
FS:  0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fdd213cb000 CR3: 00000000019ba000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88042c125d30 0000000000000048 ffff880182283b58 ffffffffa0092729
ffff88042c125008 0400000082283b18 000000000000000d ffff88042a675340
0000000000000034 ffff88042c125188 000000600000000d ffff880400000007
Call Trace:
[<ffffffffa0092729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa01572b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff814518da>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8145a9e1>] scsi_request_fn+0x501/0x770
[<ffffffff812eb053>] __blk_run_queue+0x33/0x40
[<ffffffff812eb0c6>] blk_run_queue+0x26/0x40
[<ffffffff81458d78>] scsi_run_queue+0x258/0x2f0
[<ffffffff8145ac90>] scsi_next_command+0x20/0x40
[<ffffffff8145ade5>] scsi_end_request+0x135/0x1e0
[<ffffffff8145b006>] scsi_io_completion+0x116/0x670
[<ffffffff814501b5>] scsi_finish_command+0xd5/0x130
[<ffffffff8145a4b2>] scsi_softirq_done+0x132/0x160
[<ffffffff812f59b0>] blk_done_softirq+0x90/0xc0
[<ffffffff8108f0ff>] __do_softirq+0xef/0x280
[<ffffffff8108f2c8>] run_ksoftirqd+0x38/0x50
[<ffffffff810b90af>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f80>] ? lg_double_unlock+0x90/0x90
[<ffffffff810b052f>] kthread+0xcf/0xe0
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81696518>] ret_from_fork+0x58/0x90
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 08