Complete report #1158732
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff816ab845
Build id: 3.10.0-693.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc005713c
Build id: 3.10.0-693.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc00571ed
Build id: 3.10.0-693.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff810a881a
Build id: 3.10.0-693.el7.x86_64
|
0x17a
|
5
|
worker_thread
|
vmlinux |
0xffffffff810a94e6
Build id: 3.10.0-693.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff810a93c0
Build id: 3.10.0-693.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
|
0xcf
|
8
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 47s! [kworker/0:1:26677]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) vmhgfs(OE) vmw_vsock_vmci_transport vsock bnep snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi ppdev snd_ac97_codec ac97_bus snd_seq aesni_intel snd_seq_device vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd snd_pcm btusb btrtl btbcm btintel bluetooth snd_timer joydev pcspkr rfkill snd soundcore vmw_vmci sg i2c_piix4 shpchp nfit libnvdimm parport_pc parport nfsd
auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ttm crc32c_intel mptspi scsi_transport_spi drm mptscsih ata_piix mptbase libata serio_raw e1000 i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 26677 Comm: kworker/0:1 Tainted: G OEL ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
Workqueue: events e1000_watchdog [e1000]
task: ffff8801972d0000 ti: ffff8801a7248000 task.ti: ffff8801a7248000
RIP: 0010:[<ffffffff816ab845>] [<ffffffff816ab845>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0000:ffff8801a724bd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000082 RCX: 0000379785f4bec2
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8801a724bd90 R08: ffff88023551f0b8 R09: dff6b2ba7291f0b0
R10: dff6b2ba7291f0b0 R11: ffffea0007f6cac0 R12: ffff8801a724bd48
R13: 0000000000000082 R14: ffff880035f36480 R15: ffffffff81098ae2
FS: 0000000000000000(0000) GS:ffff880235600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9339f8ba90 CR3: 000000007f500000 CR4: 00000000003407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8801a724bdd0 ffffffffc005713c 0000880230489000 00000000fe039281
ffff88023551f0b0 ffff88023551e8c0 ffff88023551e000 ffff880035d43880
ffff8801a724be18 ffffffffc00571ed ffff880035f36460 00000000fe039281
Call Trace:
[<ffffffffc005713c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc00571ed>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff810a881a>] process_one_work+0x17a/0x440
[<ffffffff810a94e6>] worker_thread+0x126/0x3c0
[<ffffffff810a93c0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Complete report #1025712
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
e1000_update_stats
|
e1000 |
0xffffffffc0214bee
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x5e
|
2
|
e1000_watchdog
|
e1000 |
0xffffffffc021558d
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x6d
|
3
|
process_one_work
|
vmlinux |
0xffffffffbbcb9d8f
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x17f
|
4
|
worker_thread
|
vmlinux |
0xffffffffbbcbae26
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x126
|
5
|
|
vmlinux |
0xffffffffbbcbad00
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x2a0
|
6
|
kthread
|
vmlinux |
0xffffffffbbcc1c71
Build id: 3.10.0-957.10.1.el7.x86_64
|
0xd1
|
7
|
|
vmlinux |
0xffffffffbbcc1ba0
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x40
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffbc375c1d
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x7
|
9
|
|
vmlinux |
0xffffffffbbcc1ba0
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 214s! [kworker/2:0:22524]
Modules linked in: veth tcp_lp fuse vboxsf(OE) xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack libcrc32c iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc dm_mirror dm_region_hash dm_log dm_mod ppdev crc32_pclmul ghash_clmulni_intel aesni_intel vboxvideo(OE) lrw gf128mul ttm glue_helper ablk_helper drm_kms_helper cryptd syscopyarea
sysfillrect sysimgblt fb_sys_fops joydev drm pcspkr sg parport_pc drm_panel_orientation_quirks parport video i2c_piix4 vboxguest(OE) ip_tables ext4 mbcache jbd2 sr_mod sd_mod crc_t10dif cdrom crct10dif_generic ata_generic pata_acpi hid_multitouch ahci crct10dif_pclmul crct10dif_common libahci crc32c_intel ata_piix libata serio_raw e1000
CPU: 2 PID: 22524 Comm: kworker/2:0 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-957.10.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
Workqueue: events e1000_watchdog [e1000]
task: ffff9d0e49031040 ti: ffff9d0e0c648000 task.ti: ffff9d0e0c648000
RIP: 0010:[<ffffffffc0214bee>] [<ffffffffc0214bee>] e1000_update_stats+0x5e/0x990 [e1000]
RSP: 0000:ffff9d0e0c64bdb0 EFLAGS: 00010282
RAX: 0000000000000000 RBX: 0000173aae114749 RCX: 0000173aae0f1ba5
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff9d0e0c64bdd0 R08: ffff9d0d7bdbb0b8 R09: 8bc9ad9eb91bb0b0
R10: 8bc9ad9eb91bb0b0 R11: 0000000000000001 R12: ffffffffbbc6a11e
R13: ffff9d0e0c64bd28 R14: 0000000000000000 R15: ffffffffbbce064c
FS: 0000000000000000(0000) GS:ffff9d0e56700000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007efc857c1010 CR3: 00000001867a4000 CR4: 00000000000206e0
Call Trace:
[<ffffffffc021558d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffbbcb9d8f>] process_one_work+0x17f/0x440
[<ffffffffbbcbae26>] worker_thread+0x126/0x3c0
[<ffffffffbbcbad00>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffffbbcc1c71>] kthread+0xd1/0xe0
[<ffffffffbbcc1ba0>] ? insert_kthread_work+0x40/0x40
[<ffffffffbc375c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffbbcc1ba0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 48 8b 87 c8 03 00 00 74 09 83 b8 90 00 00 00 01 74 29 48 8b 45 d8 65 48 33 04 25 28 00 00 00 0f 85 23 09 00 00 48 83 c4 10 <5b> 41 5c 41 5d 41 5e 5d c3 66 0f 1f 84 00 00 00 00 00 4c 8d af
Complete report #932729
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa016a0f5
Build id: 3.10.0-957.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc01374dc
Build id: 3.10.0-957.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc013758d
Build id: 3.10.0-957.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff9fab9d4f
Build id: 3.10.0-957.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffff9fabade6
Build id: 3.10.0-957.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff9fabacc0
Build id: 3.10.0-957.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff9fac1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff9fac1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa0174c1d
Build id: 3.10.0-957.el7.x86_64
|
0x7
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [kworker/0:3:41292]
Modules linked in: xt_CHECKSUM xt_nat xt_addrtype ipt_MASQUERADE nf_nat_masquerade_ipv4 ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables veth br_netfilter overlay(T) tcp_lp rfcomm fuse tun devlink ip_set nfnetlink bridge stp llc bnep vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper
ablk_helper cryptd joydev pcspkr btusb btrtl btbcm btintel bluetooth snd_ens1371 snd_rawmidi snd_ac97_codec rfkill ac97_bus snd_seq snd_seq_device snd_pcm sg snd_timer snd soundcore vmw_vmci i2c_piix4 parport_pc parport xfs libcrc32c sd_mod crc_t10dif sr_mod cdrom crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ata_generic pata_acpi serio_raw vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm e1000 ata_piix mptspi scsi_transport_spi mptscsih libata mptbase drm_panel_orientation_quirks nfit libnvdimm [last unloaded: ip_tables]
CPU: 0 PID: 41292 Comm: kworker/0:3 Kdump: loaded Tainted: G L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
Workqueue: events e1000_watchdog [e1000]
task: ffff9fa78bd32080 ti: ffff9fa7332c8000 task.ti: ffff9fa7332c8000
RIP: 0010:[<ffffffffa016a0f5>] [<ffffffffa016a0f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9fa7332cbd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffff9fbc323d RCX: 00000558a9847860
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff9fa7332cbd90 R08: ffff9fa78b9910b8 R09: 8161714348d910b0
R10: 8161714348d910b0 R11: fffff6e4804b4f00 R12: fffff6e48c224200
R13: ffff9fa5b8e65530 R14: fffff6e48c224200 R15: ffffffff9fc715ce
FS: 0000000000000000(0000) GS:ffff9fa78ba00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000032341e8 CR3: 00000003642c4000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc01374dc>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc013758d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff9fab9d4f>] process_one_work+0x17f/0x440
[<ffffffff9fabade6>] worker_thread+0x126/0x3c0
[<ffffffff9fabacc0>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffff9fac1c31>] kthread+0xd1/0xe0
[<ffffffff9fac1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa0174c1d>] ret_from_fork_nospec_begin+0x7/0x21
Complete report #641189
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
e1000_xmit_frame
|
e1000 |
0xffffffffc03fb867
Build id: 3.10.0-862.el7.x86_64
|
0x7f7
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc040045c
Build id: 3.10.0-862.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc040050d
Build id: 3.10.0-862.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff81cb2dff
Build id: 3.10.0-862.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffff81cb3ac6
Build id: 3.10.0-862.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff81cb39a0
Build id: 3.10.0-862.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff81cbae31
Build id: 3.10.0-862.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff81cbad60
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8231f61d
Build id: 3.10.0-862.el7.x86_64
|
0x7
|
10
|
|
vmlinux |
0xffffffff81cbad60
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 117s! [kube-controller:7909]
Modules linked in: veth vxlan ip6_udp_tunnel udp_tunnel xt_set ip_set_bitmap_port ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set dummy iptable_mangle xt_comment xt_mark ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 iosf_mbi crc32_pclmul snd_rawmidi ghash_clmulni_intel snd_ac97_codec aesni_intel ac97_bus lrw gf128mul snd_seq glue_helper ppdev btusb btrtl ablk_helper btbcm cryptd btintel snd_seq_device snd_pcm bluetooth snd_timer snd soundcore joydev vmw_balloon rfkill pcspkr nfit sg libnvdimm parport_pc parport shpchp i2c_piix4 vmw_vmci nf_conntrack_ipv4 nf_defrag_ipv4
ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ttm crc32c_intel drm serio_raw ata_piix mptspi scsi_transport_spi libata mptscsih e1000 mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 7909 Comm: kube-controller Kdump: loaded Tainted: G ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8892dc78dee0 ti: ffff8892873f4000 task.ti: ffff8892873f4000
RIP: 0010:[<ffffffffc03fb867>] [<ffffffffc03fb867>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 117s! [kworker/0:0:35352]
Modules linked in: veth vxlan ip6_udp_tunnel udp_tunnel xt_set ip_set_bitmap_port ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set dummy iptable_mangle xt_comment xt_mark ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 iosf_mbi crc32_pclmul snd_rawmidi ghash_clmulni_intel snd_ac97_codec aesni_intel ac97_bus lrw gf128mul snd_seq glue_helper ppdev btusb btrtl ablk_helper btbcm cryptd btintel snd_seq_device snd_pcm bluetooth snd_timer snd soundcore joydev vmw_balloon rfkill pcspkr nfit sg libnvdimm parport_pc parport shpchp i2c_piix4 vmw_vmci nf_conntrack_ipv4 nf_defrag_ipv4
ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ttm crc32c_intel drm serio_raw ata_piix mptspi scsi_transport_spi libata mptscsih e1000 mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 35352 Comm: kworker/0:0 Kdump: loaded Tainted: G ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
Workqueue: events e1000_watchdog [e1000]
task: ffff8892a1a32f70 ti: ffff88928e1d4000 task.ti: ffff88928e1d4000
RIP: 0010:[<ffffffff82315375>] [<ffffffff82315375>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff88928e1d7d90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffff81db38e6 RCX: 00000f038b04ab5a
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff88928e1d7d90 R08: ffff8892f8ba10b8 R09: ddb3a5aab5fa10b0
R10: ddb3a5aab5fa10b0 R11: ffffcc77802c4900 R12: ffffffff81ca2f1b
R13: ffffffff81cd865c R14: ffff88928e1d7d40 R15: 0000000000000001
FS: 0000000000000000(0000) GS:ffff8892fb600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6e85f37000 CR3: 0000000079274000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc040045c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc040050d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff81cb2dff>] process_one_work+0x17f/0x440
[<ffffffff81cb3ac6>] worker_thread+0x126/0x3c0
[<ffffffff81cb39a0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff81cbae31>] kthread+0xd1/0xe0
[<ffffffff81cbad60>] ? insert_kthread_work+0x40/0x40
[<ffffffff8231f61d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff81cbad60>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #609830
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffaf18b795
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc00ff60c
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc00ff6bd
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffaeabde8f
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffaeabefa6
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffaeabee80
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffaeac5e61
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffaeac5d90
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffaf195df7
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffffaf195dc8
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x48
|
11
|
|
vmlinux |
0xffffffffaeac5d90
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [kworker/1:0:21618]
Modules linked in: ppp_deflate bsd_comp ppp_async crc_ccitt ppp_generic slhc tcp_lp nls_utf8 isofs vboxsf(OE) nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay(T) binfmt_misc iosf_mbi crc32_pclmul ghash_clmulni_intel snd_intel8x0 joydev aesni_intel snd_ac97_codec ppdev ac97_bus snd_seq vboxguest(OE) lrw snd_seq_device gf128mul snd_pcm glue_helper ablk_helper cryptd sg snd_timer pcspkr snd i2c_piix4 soundcore parport_pc video parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif
crct10dif_generic sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel e1000 ahci ata_piix serio_raw libahci drm_panel_orientation_quirks libata dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 21618 Comm: kworker/1:0 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.45.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
Workqueue: events e1000_watchdog [e1000]
task: ffffa0f2b9e28000 ti: ffffa0f26398c000 task.ti: ffffa0f26398c000
RIP: 0010:[<ffffffffaf18b795>] [<ffffffffaf18b795>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0000:ffffa0f26398fd90 EFLAGS: 00010246
RAX: 0000000000000001 RBX: ffffffffaebdf0a6 RCX: 0001f83d5f2bca44
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffffa0f26398fd90 R08: ffffa0f2c55d70b8 R09: 7c348b7d029d70b0
R10: 7c348b7d029d70b0 R11: 0000000000000000 R12: ffffa0f26398fd28
R13: 000005ed55da71e6 R14: ffffffffaeae482c R15: ffffa0f26398fd28
FS: 0000000000000000(0000) GS:ffffa0f31f300000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000000000b8 CR3: 0000000035c24000 CR4: 00000000000206e0
Call Trace:
[<ffffffffc00ff60c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc00ff6bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffaeabde8f>] process_one_work+0x17f/0x440
[<ffffffffaeabefa6>] worker_thread+0x126/0x3c0
[<ffffffffaeabee80>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffaeac5e61>] kthread+0xd1/0xe0
[<ffffffffaeac5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffffaf195df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffaf195dc8>] ? ret_from_fork+0x48/0x56
[<ffffffffaeac5d90>] ? insert_kthread_work+0x40/0x40
Code: 06 bd 98 ff 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 e8 e6 bc 98 ff 66 90 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #564359
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
e1000_update_stats
|
e1000 |
0xffffffffc03f960c
Build id: 3.10.0-1160.el7.x86_64
|
0x94c
|
2
|
e1000_watchdog
|
e1000 |
0xffffffffc03f96bd
Build id: 3.10.0-1160.el7.x86_64
|
0x6d
|
3
|
process_one_work
|
vmlinux |
0xffffffffb6cbdc4f
Build id: 3.10.0-1160.el7.x86_64
|
0x17f
|
4
|
worker_thread
|
vmlinux |
0xffffffffb6cbed66
Build id: 3.10.0-1160.el7.x86_64
|
0x126
|
5
|
|
vmlinux |
0xffffffffb6cbec40
Build id: 3.10.0-1160.el7.x86_64
|
0x2a0
|
6
|
kthread
|
vmlinux |
0xffffffffb6cc5c21
Build id: 3.10.0-1160.el7.x86_64
|
0xd1
|
7
|
|
vmlinux |
0xffffffffb6cc5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb7393df7
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
9
|
|
vmlinux |
0xffffffffb6cc5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ksoftirqd/3:24]
[<ffffffffc03f960c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc03f96bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffb6cbdc4f>] process_one_work+0x17f/0x440
[<ffffffffb6cbed66>] worker_thread+0x126/0x3c0
[<ffffffffb6cbec40>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffb6cc5c21>] kthread+0xd1/0xe0
[<ffffffffb6cc5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffb7393df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb6cc5b50>] ? insert_kthread_work+0x40/0x40
Complete report #491848
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff9f38ea15
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc039760c
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc03976bd
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff9ecbdfbf
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffff9ecbf0d6
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff9ecbefb0
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff9ecc5f91
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xd1
|
8
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff9f399ec9
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x96
|
9
|
|
vmlinux |
0xffffffff9ecc5ec0
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x40
|
10
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff9f399df7
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x21
|
11
|
|
vmlinux |
0xffffffff9ecc5ec0
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 65s! [kworker/1:0:4234]
Modules linked in: ppp_deflate bsd_comp ppp_async crc_ccitt ppp_generic slhc tcp_lp nls_utf8 isofs vboxsf(OE) nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter binfmt_misc overlay(T) snd_intel8x0 joydev snd_ac97_codec ac97_bus ppdev snd_seq snd_seq_device vboxguest(OE) snd_pcm sg iosf_mbi pcspkr snd_timer i2c_piix4 snd parport_pc parport video soundcore nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm drm_panel_orientation_quirks crc32c_intel serio_raw ahci libahci ata_piix libata e1000 dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 4234 Comm: kworker/1:0 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.71.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
Workqueue: events e1000_watchdog [e1000]
task: ffff9a3c7312c200 ti: ffff9a3d15d80000 task.ti: ffff9a3d15d80000
RIP: 0010:[<ffffffff9f38ea15>] [<ffffffff9f38ea15>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9a3d15d83d90 EFLAGS: 00010246
RAX: 0000000000000001 RBX: ffff9a3c7312c268 RCX: 000045d0bc993e45
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff9a3d15d83d90 R08: ffff9a3d4f0970b8 R09: 970a5e178c4970b0
R10: 970a5e178c4970b0 R11: 0000000000000005 R12: ffff9a3d15d83d28
R13: 000001376c0cc884 R14: ffffffff9ece495c R15: ffff9a3d15d83d28
FS: 0000000000000000(0000) GS:ffff9a3d56d00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fed6b623000 CR3: 000000018e168000 CR4: 00000000000206e0
Call Trace:
[<ffffffffc039760c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc03976bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff9ecbdfbf>] process_one_work+0x17f/0x440
[<ffffffff9ecbf0d6>] worker_thread+0x126/0x3c0
[<ffffffff9ecbefb0>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff9ecc5f91>] kthread+0xd1/0xe0
[<ffffffff9f399ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff9ecc5ec0>] ? insert_kthread_work+0x40/0x40
[<ffffffff9f399df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9ecc5ec0>] ? insert_kthread_work+0x40/0x40
Code: b6 8b 98 ff 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 e8 96 8b 98 ff 66 90 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #446604
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb1589785
Build id: 3.10.0-1160.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc01df60c
Build id: 3.10.0-1160.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc01df6bd
Build id: 3.10.0-1160.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffb0ebdc4f
Build id: 3.10.0-1160.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffb0ebed66
Build id: 3.10.0-1160.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffb0ebec40
Build id: 3.10.0-1160.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffb0ec5c21
Build id: 3.10.0-1160.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffb0ec5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb1593df7
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffffb0ec5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 31s! [kworker/0:2:77083]
Modules linked in: veth br_netfilter tun devlink overlay(T) bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec iosf_mbi crc32_pclmul ac97_bus ghash_clmulni_intel snd_seq snd_seq_device aesni_intel snd_pcm lrw gf128mul ppdev glue_helper ablk_helper cryptd snd_timer vmw_balloon snd soundcore joydev sg pcspkr i2c_piix4 vmw_vmci parport_pc parport xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common syscopyarea sysfillrect crc32c_intel sysimgblt fb_sys_fops ttm nfit libnvdimm drm serio_raw mptspi e1000 scsi_transport_spi ata_piix mptscsih libata mptbase drm_panel_orientation_quirks
fuse [last unloaded: ip_tables]
CPU: 0 PID: 77083 Comm: kworker/0:2 Kdump: loaded Tainted: G B L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
Workqueue: events e1000_watchdog [e1000]
task: ffff9c8581d2d280 ti: ffff9c8486c18000 task.ti: ffff9c8486c18000
RIP: 0010:[<ffffffffb1589785>] [<ffffffffb1589785>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9c8486c1bd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff9c8486c1bd60 RCX: 003e609dc5af1180
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff9c8486c1bd90 R08: ffff9c85afd670b8 R09: 8de8dd2bed1670b0
R10: 8de8dd2bed1670b0 R11: 00007f4c89bcf000 R12: ffffffffc0303578
R13: ffffffffb0ead8b3 R14: ffff9c8486c1bd10 R15: ffffffffc0303578
FS: 0000000000000000(0000) GS:ffff9c85b9600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4cc31b9500 CR3: 0000000063c10000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc01df60c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc01df6bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffb0ebdc4f>] process_one_work+0x17f/0x440
[<ffffffffb0ebed66>] worker_thread+0x126/0x3c0
[<ffffffffb0ebec40>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffb0ec5c21>] kthread+0xd1/0xe0
[<ffffffffb0ec5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffb1593df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb0ec5b50>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #372628
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa3d15375
Build id: 3.10.0-862.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc03d745c
Build id: 3.10.0-862.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc03d750d
Build id: 3.10.0-862.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffa36b2dff
Build id: 3.10.0-862.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffa36b3ac6
Build id: 3.10.0-862.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffa36b39a0
Build id: 3.10.0-862.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffa36bae31
Build id: 3.10.0-862.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffa36bad60
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa3d1f637
Build id: 3.10.0-862.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffffa36bad60
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [kworker/0:1:119]
Modules linked in: tcp_lp fuse vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq crc32_pclmul ghash_clmulni_intel aesni_intel snd_seq_device lrw snd_pcm snd_timer snd gf128mul soundcore joydev pcspkr nfit ppdev glue_helper ablk_helper sg cryptd parport_pc parport libnvdimm vmw_balloon shpchp vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm crct10dif_pclmul crct10dif_common libata mptspi crc32c_intel scsi_transport_spi e1000 mptscsih serio_raw mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 119 Comm: kworker/0:1 Kdump: loaded Tainted: G W L ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
Workqueue: events e1000_watchdog [e1000]
task: ffff9f27f5e7dee0 ti: ffff9f27f5a64000 task.ti: ffff9f27f5a64000
RIP: 0010:[<ffffffffa3d15375>] [<ffffffffa3d15375>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9f27f5a67d90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffffa37b38e6 RCX: 000016686337bb2f
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff9f27f5a67d90 R08: ffff9f28380090b8 R09: 835ebf25f54090b0
R10: 835ebf25f54090b0 R11: 0000000000000000 R12: ffffffffa36a2f1b
R13: ffffffffa36d865c R14: ffff9f27f5a67d40 R15: 0000000000000001
FS: 0000000000000000(0000) GS:ffff9f283b600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f25d1124a08 CR3: 000000005340e000 CR4: 00000000007607f0
PKRU: 00000000
Call Trace:
[<ffffffffc03d745c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc03d750d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffa36b2dff>] process_one_work+0x17f/0x440
[<ffffffffa36b3ac6>] worker_thread+0x126/0x3c0
[<ffffffffa36b39a0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffffa36bae31>] kthread+0xd1/0xe0
[<ffffffffa36bad60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa3d1f637>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa36bad60>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #369907
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff816adce5
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc006613c
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc00661ed
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff810aa59a
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x17a
|
5
|
worker_thread
|
vmlinux |
0xffffffff810ab266
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff810ab140
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff810b270f
Build id: 3.10.0-693.17.1.el7.x86_64
|
0xcf
|
8
|
|
vmlinux |
0xffffffff810b2640
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x40
|
9
|
ret_from_fork
|
vmlinux |
0xffffffff816b8798
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x58
|
10
|
|
vmlinux |
0xffffffff810b2640
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 30s! [kworker/0:0:58215]
Modules linked in: fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc nfsv3 ebtable_filter nfs fscache ebtables ip6table_filter ip6_tables iptable_filter mvfs(OE) vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev snd_ens1371 snd_rawmidi aesni_intel snd_ac97_codec lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon ac97_bus snd_seq snd_seq_device pcspkr snd_pcm joydev sg nfit snd_timer libnvdimm snd soundcore shpchp vmw_vmci i2c_piix4 parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom
ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm serio_raw ata_piix mptspi libata scsi_transport_spi mptscsih e1000 mptbase i2c_core floppy
CPU: 0 PID: 58215 Comm: kworker/0:0 Tainted: G W OEL ------------ 3.10.0-693.17.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
Workqueue: events e1000_watchdog [e1000]
task: ffff8801802a6eb0 ti: ffff88021fa78000 task.ti: ffff88021fa78000
RIP: 0010:[<ffffffff816adce5>] [<ffffffff816adce5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff88021fa7bd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff88021fa7bd20 RCX: 001d45b652b73a4f
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff88021fa7bd90 R08: ffff880235cf90b8 R09: dff6b0c3f30f90b0
R10: dff6b0c3f30f90b0 R11: 0000000000000000 R12: 0000000000000001
R13: ffffffff810cfb6c R14: ffff88021fa7bd40 R15: 0000000000000001
FS: 0000000000000000(0000) GS:ffff880236600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fe0c3bfe000 CR3: 000000022eea0000 CR4: 00000000001607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc006613c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc00661ed>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff810aa59a>] process_one_work+0x17a/0x440
[<ffffffff810ab266>] worker_thread+0x126/0x3c0
[<ffffffff810ab140>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b270f>] kthread+0xcf/0xe0
[<ffffffff810b2640>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b8798>] ret_from_fork+0x58/0x90
[<ffffffff810b2640>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #368719
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb15817c5
Build id: 3.10.0-1062.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc030760c
Build id: 3.10.0-1062.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc03076bd
Build id: 3.10.0-1062.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffb0ebd0ff
Build id: 3.10.0-1062.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffb0ebe216
Build id: 3.10.0-1062.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffb0ebe0f0
Build id: 3.10.0-1062.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffb0ec50d1
Build id: 3.10.0-1062.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffb0ed3f07
Build id: 3.10.0-1062.el7.x86_64
|
0x57
|
9
|
|
vmlinux |
0xffffffffb0ec5000
Build id: 3.10.0-1062.el7.x86_64
|
0x40
|
10
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb158bd37
Build id: 3.10.0-1062.el7.x86_64
|
0x21
|
11
|
|
vmlinux |
0xffffffffb0ec5000
Build id: 3.10.0-1062.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 25s! [kworker/1:2:127389]
Modules linked in: nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq ppdev vmw_balloon snd_seq_device snd_pcm snd_timer snd soundcore sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul vmw_vmci pcspkr glue_helper joydev ablk_helper cryptd sg i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit libnvdimm drm ata_piix libata mptspi crct10dif_pclmul crct10dif_common scsi_transport_spi mptscsih crc32c_intel e1000 mptbase serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 127389 Comm: kworker/1:2 Kdump: loaded Tainted: G L ------------ T 3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
Workqueue: events e1000_watchdog [e1000]
task: ffff8a565ee7d230 ti: ffff8a5a693c4000 task.ti: ffff8a5a693c4000
RIP: 0010:[<ffffffffb15817c5>] [<ffffffffb15817c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8a5a693c7d90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffffb0fdc446 RCX: 0004e8b3ebdfa1f6
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8a5a693c7d90 R08: ffff8a587f33f0b8 R09: d69d8d88bc73f0b0
R10: d69d8d88bc73f0b0 R11: 0000000000000000 R12: ffff8a5a693c7d50
R13: 0000000000000001 R14: ffff8a5a6d65acc0 R15: ffff8a565ee7d298
FS: 0000000000000000(0000) GS:ffff8a5a6d640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3743522000 CR3: 000000035c700000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc030760c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc03076bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffb0ebd0ff>] process_one_work+0x17f/0x440
[<ffffffffb0ebe216>] worker_thread+0x126/0x3c0
[<ffffffffb0ebe0f0>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffb0ec50d1>] kthread+0xd1/0xe0
[<ffffffffb0ed3f07>] ? finish_task_switch+0x57/0x1c0
[<ffffffffb0ec5000>] ? insert_kthread_work+0x40/0x40
[<ffffffffb158bd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb0ec5000>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #340142
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff88b83815
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc03d960c
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc03d96bd
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x6d
|
4
|
|
vmlinux |
0xffffffff88a35e53
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x23
|
5
|
process_one_work
|
vmlinux |
0xffffffff884be34f
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x17f
|
6
|
worker_thread
|
vmlinux |
0xffffffff884bf466
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x126
|
7
|
|
vmlinux |
0xffffffff884bf340
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x2a0
|
8
|
kthread
|
vmlinux |
0xffffffff884c6321
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xd1
|
9
|
|
vmlinux |
0xffffffff884c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x40
|
10
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff88b8dd37
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x21
|
11
|
|
vmlinux |
0xffffffff884c6250
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 28s! [kworker/0:1:119209]
Modules linked in: binfmt_misc snd_seq_midi snd_seq_midi_event vmw_vsock_vmci_transport vsock iosf_mbi crc32_pclmul snd_ens1371 ghash_clmulni_intel snd_rawmidi snd_ac97_codec ppdev ac97_bus snd_seq aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd snd_seq_device snd_pcm snd_timer joydev pcspkr sg snd soundcore i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper ata_piix syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common crc32c_intel ttm nfit libata libnvdimm mptspi scsi_transport_spi mptscsih drm mptbase e1000 serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 119209 Comm: kworker/0:1 Kdump: loaded Tainted: G L ------------ 3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
ata2.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x6 frozen
ata2.00: cmd a0/00:00:00:08:00/00:00:00:00:00/a0 tag 0 pio 16392 in#012 Get event status notification 4a 01 00 00 10 00 00 00 08 00res 40/00:02:00:08:00/00:00:00:00:00/a0 Emask 0x4 (timeout)
ata2.00: status: { DRDY }
ata2: soft resetting link
Workqueue: events e1000_watchdog [e1000]
task: ffff96501fff62a0 ti: ffff965035aa8000 task.ti: ffff965035aa8000
RIP: 0010:[<ffffffff88b83815>] [<ffffffff88b83815>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff965035aabd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000001400 RCX: 0005054cb427fe72
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff965035aabd90 R08: ffff9650350550b8 R09: a6bec23af24550b0
R10: a6bec23af24550b0 R11: ffffded282e0c800 R12: ffff96503961f400
R13: 0000000200000000 R14: ffffffff884bc10c R15: ffff965035aabd20
FS: 0000000000000000(0000) GS:ffff965039600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f25ba763000 CR3: 000000007dbd0000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc03d960c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc03d96bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff88a35e53>] ? sk_destruct+0x23/0x40
[<ffffffff884be34f>] process_one_work+0x17f/0x440
[<ffffffff884bf466>] worker_thread+0x126/0x3c0
[<ffffffff884bf340>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff884c6321>] kthread+0xd1/0xe0
[<ffffffff884c6250>] ? insert_kthread_work+0x40/0x40
[<ffffffff88b8dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff884c6250>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #321778
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa9d8b495
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc022f60c
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc022f6bd
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffa96bddcf
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffa96beee6
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffa96bedc0
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffa96c5da1
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffa96c5cd0
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa9d95df7
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffffa96c5cd0
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [kworker/0:0:23163]
Modules linked in: talpa_vfshook(OE) talpa_pedconnector(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) talpa_pedevice(OE) tcp_diag inet_diag xt_connlimit ip6t_REJECT nf_reject_ipv6 nf_log_ipv6 ip6table_filter nf_log_ipv4 nf_log_common nf_nat_ftp xt_REDIRECT nf_nat_redirect xt_conntrack nf_conntrack_ftp xt_LOG xt_limit xt_multiport iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 ip6table_mangle ip6table_raw ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 nf_nat nf_conntrack ip6_tables iptable_mangle iptable_raw ipt_REJECT nf_reject_ipv4 iptable_filter sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd sg vmw_balloon pcspkr joydev parport_pc vmw_vmci parport i2c_piix4 tcp_htcp
ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm sd_mod crc_t10dif crct10dif_generic drm ata_piix crct10dif_pclmul crct10dif_common libata crc32c_intel serio_raw e1000 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: talpa_syscallhook]
CPU: 0 PID: 23163 Comm: kworker/0:0 Kdump: loaded Tainted: G W OEL ------------ 3.10.0-1160.21.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
Workqueue: events e1000_watchdog [e1000]
task: ffff8a4f843bb180 ti: ffff8a4f80bdc000 task.ti: ffff8a4f80bdc000
RIP: 0010:[<ffffffffa9d8b495>] [<ffffffffa9d8b495>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0000:ffff8a4f80bdfd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffffaa704bc0 RCX: 013ff7b90fe3b815
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8a4f80bdfd90 R08: ffff8a4f283db0b8 R09: d6c2e958657db0b0
R10: d6c2e958657db0b0 R11: 0000000000000000 R12: ffff8a4f80bdfdd0
R13: 0000000000000001 R14: ffff8a4f3c780000 R15: 000000000000002f
FS: 0000000000000000(0000) GS:ffff8a51ffc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000562eb6604688 CR3: 0000000035fe0000 CR4: 00000000000407f0
Call Trace:
[<ffffffffc022f60c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc022f6bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffa96bddcf>] process_one_work+0x17f/0x440
[<ffffffffa96beee6>] worker_thread+0x126/0x3c0
[<ffffffffa96bedc0>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffa96c5da1>] kthread+0xd1/0xe0
[<ffffffffa96c5cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffffa9d95df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa96c5cd0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #234549
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff9d36b455
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc013e4dc
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc013e58d
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff9ccb9ebf
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x17f
|
5
|
process_one_work
|
vmlinux |
0xffffffff9ccb9ebf
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x17f
|
6
|
|
vmlinux |
0xffffffff9ccbae30
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff9ccc1da1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff9ccc1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff9d375c37
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffff9ccc1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 183s! [kworker/0:2:22607]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nf_conntrack_netbios_ns nf_conntrack_broadcast nf_nat_ftp nf_conntrack_ftp ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac coretemp iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon snd_seq_midi snd_seq_midi_event pcspkr joydev
snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore sg parport_pc parport vmw_vmci i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ata_generic serio_raw pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm mptspi scsi_transport_spi e1000 libata mptscsih mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 22607 Comm: kworker/0:2 Kdump: loaded Tainted: G L ------------ 3.10.0-957.21.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/20/2014
Workqueue: events e1000_watchdog [e1000]
task: ffff8ae9e3d530c0 ti: ffff8aea64e10000 task.ti: ffff8aea64e10000
RIP: 0010:[<ffffffff9d36b455>] [<ffffffff9d36b455>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8aea64e13d90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000200000000 RCX: 000012906e70405b
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8aea64e13d90 R08: ffff8aeaf646b0b8 R09: d453b1d03386b0b0
R10: d453b1d03386b0b0 R11: 0000000000000000 R12: ffff8aeafa61ab80
R13: 0000000000000000 R14: 0000000000000000 R15: ffff8aeaf85be800
FS: 0000000000000000(0000) GS:ffff8aeafa600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f1250c3c000 CR3: 00000000b0130000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc013e4dc>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc013e58d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff9ccb9ebf>] process_one_work+0x17f/0x440
[<ffffffff9ccb9ebf>] process_one_work+0x17f/0x440
[<ffffffff9ccbae30>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffff9ccc1da1>] kthread+0xd1/0xe0
[<ffffffff9ccc1cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffff9d375c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9ccc1cd0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #194266
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffaf78b805
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc03bc60c
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc03bc6bd
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffaf0bde5f
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffaf0bef76
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffaf0bee50
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffaf0c5e31
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xd1
|
8
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffaf795ec9
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x96
|
9
|
|
vmlinux |
0xffffffffaf0c5d60
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x40
|
10
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffaf795df7
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x21
|
11
|
|
vmlinux |
0xffffffffaf0c5d60
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 56s! [kworker/0:0:16371]
Modules linked in: binfmt_misc tun xt_hl ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel aesni_intel snd_ens1371 lrw gf128mul glue_helper ablk_helper snd_rawmidi cryptd snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer ppdev snd soundcore pcspkr joydev vmw_balloon sg vmw_vmci i2c_piix4 parport_pc parport
ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ttm crc32c_intel drm ata_piix nfit mptspi libata serio_raw libnvdimm scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 16371 Comm: kworker/0:0 Tainted: G L ------------ 3.10.0-1160.31.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
Workqueue: events e1000_watchdog [e1000]
task: ffff97dbd8ff8000 ti: ffff97db6c328000 task.ti: ffff97db6c328000
RIP: 0010:[<ffffffffaf78b805>] [<ffffffffaf78b805>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0000:ffff97db6c32bd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff97dbd8ff8068 RCX: 00023717e08c29d4
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff97db6c32bd90 R08: ffff97dbdbd330b8 R09: a090288f191330b0
R10: a090288f191330b0 R11: 0000000000000005 R12: ffff97db6c32bd28
R13: 000034851245c0ec R14: ffffffffaf0e452c R15: ffff97db6c32bd28
FS: 0000000000000000(0000) GS:ffff97dbdd600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002357cdb20000 CR3: 000000077284a000 CR4: 00000000000007f0
Call Trace:
[<ffffffffc03bc60c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc03bc6bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffaf0bde5f>] process_one_work+0x17f/0x440
[<ffffffffaf0bef76>] worker_thread+0x126/0x3c0
[<ffffffffaf0bee50>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffaf0c5e31>] kthread+0xd1/0xe0
[<ffffffffaf795ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffffaf0c5d60>] ? insert_kthread_work+0x40/0x40
[<ffffffffaf795df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffaf0c5d60>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #149290
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb616a0f5
Build id: 3.10.0-957.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc05634dc
Build id: 3.10.0-957.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc056358d
Build id: 3.10.0-957.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffb5ab9d4f
Build id: 3.10.0-957.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffb5abade6
Build id: 3.10.0-957.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffb5abacc0
Build id: 3.10.0-957.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffb5ac1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffb5ad0887
Build id: 3.10.0-957.el7.x86_64
|
0x57
|
9
|
|
vmlinux |
0xffffffffb5ac1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
10
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb6174c1d
Build id: 3.10.0-957.el7.x86_64
|
0x7
|
11
|
|
vmlinux |
0xffffffffb5ac1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 37s! [kworker/1:2:29264]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq joydev pcspkr snd_seq_device snd_pcm snd_timer sg snd soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables ext4 mbcache jbd2 sd_mod sr_mod crc_t10dif cdrom crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel
serio_raw vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt e1000 fb_sys_fops ttm mptspi drm nfit ata_piix libnvdimm scsi_transport_spi mptscsih libata mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 29264 Comm: kworker/1:2 Kdump: loaded Tainted: G L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
Workqueue: events e1000_watchdog [e1000]
task: ffff9e820735a080 ti: ffff9e821ba1c000 task.ti: ffff9e821ba1c000
RIP: 0010:[<ffffffffb616a0f5>] [<ffffffffb616a0f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9e821ba1fd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000200000000 RCX: 00010489de43490b
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff9e821ba1fd90 R08: ffff9e82304c70b8 R09: 85f6dd506d8c70b0
R10: 85f6dd506d8c70b0 R11: 0000000000000000 R12: ffff9e823365ab80
R13: 0000000100000000 R14: ffff9e823361ab80 R15: ffff9e80fb16d000
FS: 0000000000000000(0000) GS:ffff9e8233640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa49937fe00 CR3: 0000000035fa2000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc05634dc>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc056358d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffb5ab9d4f>] process_one_work+0x17f/0x440
[<ffffffffb5abade6>] worker_thread+0x126/0x3c0
[<ffffffffb5abacc0>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffffb5ac1c31>] kthread+0xd1/0xe0
[<ffffffffb5ad0887>] ? finish_task_switch+0x57/0x1c0
[<ffffffffb5ac1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffffb6174c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffb5ac1b60>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #116948
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff816ab915
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc007913c
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc00791ed
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff810a882a
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x17a
|
5
|
worker_thread
|
vmlinux |
0xffffffff810a94f6
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff810a93d0
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff810b099f
Build id: 3.10.0-693.11.1.el7.x86_64
|
0xcf
|
8
|
|
vmlinux |
0xffffffff810b08d0
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x40
|
9
|
ret_from_fork
|
vmlinux |
0xffffffff816b4fd8
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x58
|
10
|
|
vmlinux |
0xffffffff810b08d0
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 25s! [kworker/1:2:101358]
Modules linked in: binfmt_misc tcp_lp isofs fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmhgfs(OE) vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul snd_ens1371 ghash_clmulni_intel snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device aesni_intel ppdev lrw gf128mul glue_helper ablk_helper cryptd snd_pcm vmw_balloon snd_timer snd soundcore joydev pcspkr nfit libnvdimm sg parport_pc parport vmw_vmci shpchp i2c_piix4 nfsd auth_rpcgss dm_multipath nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi
sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common crc32c_intel syscopyarea sysfillrect sysimgblt fb_sys_fops ttm serio_raw drm ata_piix mptspi scsi_transport_spi libata mptscsih e1000 mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 101358 Comm: kworker/1:2 Tainted: G OE ------------ 3.10.0-693.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
Workqueue: events e1000_watchdog [e1000]
task: ffff880035b4eeb0 ti: ffff880050ea4000 task.ti: ffff880050ea4000
RIP: 0010:[<ffffffff816ab915>] [<ffffffff816ab915>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff880050ea7d90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000086 RCX: 0005de61f8d9d777
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff880050ea7d90 R08: ffff880078b150b8 R09: dffda53b35f150b0
R10: dffda53b35f150b0 R11: 0000000000000000 R12: ffffffff81098af2
R13: ffff880050ea7d18 R14: ffffffff81098a3b R15: ffff880050ea7cf8
FS: 0000000000000000(0000) GS:ffff88007b640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f81fb208000 CR3: 00000000769c5000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff880050ea7dd0 ffffffffc007913c 0000ffffc04bc3c0 000000009b65e95b
ffff880078b150b0 ffff880078b148c0 ffff880078b14000 ffff8800356e0140
ffff880050ea7e18 ffffffffc00791ed 0000000144964500 000000009b65e95b
Call Trace:
[<ffffffffc007913c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc00791ed>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff810a882a>] process_one_work+0x17a/0x440
[<ffffffff810a94f6>] worker_thread+0x126/0x3c0
[<ffffffff810a93d0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b099f>] kthread+0xcf/0xe0
[<ffffffff810b08d0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4fd8>] ret_from_fork+0x58/0x90
[<ffffffff810b08d0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #91448
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff816ab845
Build id: 3.10.0-693.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc009d13c
Build id: 3.10.0-693.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc009d1ed
Build id: 3.10.0-693.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff810a881a
Build id: 3.10.0-693.el7.x86_64
|
0x17a
|
5
|
worker_thread
|
vmlinux |
0xffffffff810a94e6
Build id: 3.10.0-693.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff810a93c0
Build id: 3.10.0-693.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
|
0xcf
|
8
|
|
vmlinux |
0xffffffff8108ddeb
Build id: 3.10.0-693.el7.x86_64
|
0x6bb
|
9
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
10
|
ret_from_fork
|
vmlinux |
0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
|
0x58
|
11
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 35s! [kworker/7:0:61030]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq coretemp snd_seq_device crc32_pclmul ghash_clmulni_intel snd_pcm aesni_intel lrw snd_timer gf128mul glue_helper snd ablk_helper cryptd sg soundcore ppdev pcspkr joydev vmw_balloon parport_pc parport vmw_vmci i2c_piix4 shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm ata_piix libata serio_raw mptspi scsi_transport_spi e1000 mptscsih mptbase i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 7 PID: 61030 Comm: kworker/7:0 Tainted: G L ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2012
Workqueue: events e1000_watchdog [e1000]
task: ffff8801b7261fa0 ti: ffff8801b2a68000 task.ti: ffff8801b2a68000
RIP: 0010:[<ffffffff816ab845>] [<ffffffff816ab845>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8801b2a6bd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff8801b2a6bd20 RCX: 000026f692659c6e
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8801b2a6bd90 R08: ffff88023f7e30b8 R09: dff68a097cbe30b0
R10: dff68a097cbe30b0 R11: 0000000000000000 R12: 0000000000000007
R13: ffffffff810be03a R14: ffff8801b2a6bd10 R15: ffff8802367d6cc0
FS: 0000000000000000(0000) GS:ffff8802367c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f74da150580 CR3: 00000000b88fb000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8801b2a6bdd0 ffffffffc009d13c 00000000810d2abd 000000009ab5985d
ffff88023f7e30b0 ffff88023f7e28c0 ffff88023f7e2000 ffff8802321b4480
ffff8801b2a6be18 ffffffffc009d1ed ffff880235533200 000000009ab5985d
Call Trace:
[<ffffffffc009d13c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc009d1ed>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff810a881a>] process_one_work+0x17a/0x440
[<ffffffff810a94e6>] worker_thread+0x126/0x3c0
[<ffffffff810a93c0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff8108ddeb>] ? do_exit+0x6bb/0xa40
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #61662
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa9f15375
Build id: 3.10.0-862.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc023d45c
Build id: 3.10.0-862.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc023d50d
Build id: 3.10.0-862.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffa98b2dff
Build id: 3.10.0-862.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffa98b3ac6
Build id: 3.10.0-862.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffa98b39a0
Build id: 3.10.0-862.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffa98bae31
Build id: 3.10.0-862.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffa98c9664
Build id: 3.10.0-862.el7.x86_64
|
0x54
|
9
|
|
vmlinux |
0xffffffffa98bad60
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
10
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa9f1f624
Build id: 3.10.0-862.el7.x86_64
|
0xe
|
11
|
|
vmlinux |
0xffffffffa98bad60
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 89s! [kworker/1:1:62954]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec edac_mce_amd ac97_bus snd_seq crc32_pclmul ghash_clmulni_intel snd_seq_device snd_pcm aesni_intel lrw snd_timer gf128mul glue_helper snd ablk_helper ppdev cryptd soundcore pcspkr vmw_vmci vmw_balloon joydev nfit sg i2c_piix4 libnvdimm parport_pc parport shpchp ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif
crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ata_piix ttm crct10dif_pclmul crct10dif_common crc32c_intel mptspi serio_raw scsi_transport_spi libata mptscsih drm e1000 mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 62954 Comm: kworker/1:1 Kdump: loaded Tainted: G L ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
Workqueue: events e1000_watchdog [e1000]
task: ffff8fb39b203f40 ti: ffff8fb300248000 task.ti: ffff8fb300248000
RIP: 0010:[<ffffffffa9f15375>] [<ffffffffa9f15375>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8fb30024bd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffffa99b38e6 RCX: 000009759826a90d
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8fb30024bd90 R08: ffff8fb2fc6ff0b8 R09: c1339df339aff0b0
R10: c1339df339aff0b0 R11: 0000000000000000 R12: ffffffffa98d57bb
R13: ffffffffa98d865c R14: ffff8fb30024bd40 R15: 0000000000000001
FS: 00007f59732d0700(0000) GS:ffff8fb3b5640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000560ffea2b600 CR3: 00000002253aa000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc023d45c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc023d50d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffa98b2dff>] process_one_work+0x17f/0x440
[<ffffffffa98b3ac6>] worker_thread+0x126/0x3c0
[<ffffffffa98b39a0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffffa98bae31>] kthread+0xd1/0xe0
[<ffffffffa98c9664>] ? finish_task_switch+0x54/0x170
[<ffffffffa98bad60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa9f1f624>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffffa98bad60>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #59465
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb7f6b455
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc021c4dc
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc021c58d
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffb78b9ebf
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffb78baf56
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffb78bae30
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffb78c1da1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffb78c1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb7f75c37
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffffb78c1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 51s! [kworker/4:2:28619]
Modules linked in: tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 xt_conntrack vmw_vsock_vmci_transport vsock iptable_filter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_security iptable_raw sunrpc sb_edac coretemp ext4 iosf_mbi crc32_pclmul mbcache ppdev ghash_clmulni_intel jbd2 aesni_intel lrw gf128mul glue_helper vmw_balloon ablk_helper cryptd pcspkr joydev sg parport_pc i2c_piix4 parport vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_generic pata_acpi e1000 vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm
drm ata_piix mptspi libata scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 28619 Comm: kworker/4:2 Kdump: loaded Tainted: G W L ------------ 3.10.0-957.21.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
Workqueue: events e1000_watchdog [e1000]
task: ffff9e717f6230c0 ti: ffff9e6e4b480000 task.ti: ffff9e6e4b480000
RIP: 0010:[<ffffffffb7f6b455>] [<ffffffffb7f6b455>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9e6e4b483d90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff9e717fd13900 RCX: 0031a3d14de61705
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff9e6e4b483d90 R08: ffff9e716eff90b8 R09: 8639e2732c3f90b0
R10: 8639e2732c3f90b0 R11: 0000000000000001 R12: ffff9e6e4b483d50
R13: 0000000000000001 R14: ffff9e717fd1ac00 R15: ffff9e717f623128
FS: 0000000000000000(0000) GS:ffff9e717fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fbd2661a000 CR3: 0000000107b94000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc021c4dc>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc021c58d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffb78b9ebf>] process_one_work+0x17f/0x440
[<ffffffffb78baf56>] worker_thread+0x126/0x3c0
[<ffffffffb78bae30>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffffb78c1da1>] kthread+0xd1/0xe0
[<ffffffffb78c1cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb7f75c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb78c1cd0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #50250
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb7b883c5
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc024060c
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc02406bd
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffb74be6bf
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffb74bf7d6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffb74bf6b0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffb74c6691
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffb74c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb7b92d37
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffffb74c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 24s! [kworker/7:1:6780]
cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common crc32c_intel mptspi scsi_transport_spi libata mptscsih serio_raw mptbase e1000 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ip_tables]
CPU: 7 PID: 6780 Comm: kworker/7:1 Kdump: loaded Tainted: P W OEL ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
Workqueue: events e1000_watchdog [e1000]
task: ffff89aac6b28000 ti: ffff89a8db674000 task.ti: ffff89a8db674000
RIP: 0010:[<ffffffffb7b883c5>] [<ffffffffb7b883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff89a8db677d90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000002 RCX: 003f7fa063b646d2
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff89a8db677d90 R08: ffff89aaf71a50b8 R09: d953ad41b45a50b0
R10: d953ad41b45a50b0 R11: ffff89aa890eff00 R12: 0000000000000000
R13: ffff89aa3fd11c00 R14: 0000000000000000 R15: 0000000200000000
FS: 0000000000000000(0000) GS:ffff89aaffdc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fecf911c000 CR3: 000000005f010000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc024060c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc02406bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffb74be6bf>] process_one_work+0x17f/0x440
[<ffffffffb74bf7d6>] worker_thread+0x126/0x3c0
[<ffffffffb74bf6b0>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffb74c6691>] kthread+0xd1/0xe0
[<ffffffffb74c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb7b92d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb74c65c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #24744
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8a71b445
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc045645c
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc045650d
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff8a0b613f
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffff8a0b71d6
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff8a0b70b0
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff8a0bdf21
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff8a0bde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8a7255f7
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffff8a0bde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 67s! [kworker/0:4:9846]
Modules linked in: uas usb_storage tcp_lp nls_utf8 isofs fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack 8021q garp mrp ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock vboxpci(OE) vboxnetadp(OE) vboxnetflt(OE) vboxdrv(OE) sunrpc snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi
aesni_intel snd_ac97_codec lrw gf128mul glue_helper ablk_helper ac97_bus cryptd ppdev snd_seq vmw_balloon snd_seq_device snd_pcm snd_timer snd nfit soundcore pcspkr joydev sg libnvdimm parport_pc parport vmw_vmci shpchp i2c_piix4 sch_fq_codel ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common ata_piix crc32c_intel mptspi libata scsi_transport_spi e1000 mptscsih serio_raw mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 9846 Comm: kworker/0:4 Kdump: loaded Tainted: G OEL ------------ 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
Workqueue: events e1000_watchdog [e1000]
task: ffff8caa83b48000 ti: ffff8cab5aa0c000 task.ti: ffff8cab5aa0c000
RIP: 0010:[<ffffffff8a71b445>] [<ffffffff8a71b445>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8cab5aa0fd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff8cab7d600000 RCX: 0000236ca763886b
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8cab5aa0fd90 R08: ffff8cab7ca570b8 R09: cd519a15b9e570b0
R10: cd519a15b9e570b0 R11: 0000000000000000 R12: ffff8cab5aa0fd58
R13: ffff8cab7d618bc0 R14: ffff8caa83b48068 R15: 0000000000000001
FS: 0000000000000000(0000) GS:ffff8cab7d600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055834ab66200 CR3: 0000000fe6fe2000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc045645c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc045650d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff8a0b613f>] process_one_work+0x17f/0x440
[<ffffffff8a0b71d6>] worker_thread+0x126/0x3c0
[<ffffffff8a0b70b0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff8a0bdf21>] kthread+0xd1/0xe0
[<ffffffff8a0bde50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8a7255f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8a0bde50>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #23282
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff81783045
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc035660c
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc03566bd
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff810be21f
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffff810bf336
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff810bf210
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff810c61f1
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff810c6120
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8178dd24
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xe
|
10
|
|
vmlinux |
0xffffffff810c6120
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 56s! [kworker/0:0:32358]
Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables bridge stp llc ip_set nfnetlink sunrpc ppdev edac_mce_amd crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg virtio_balloon joydev parport_pc i2c_piix4 parport pcspkr xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_blk vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common
drm ata_piix virtio_pci virtio_ring crc32c_intel libata e1000 serio_raw virtio drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ip_tables]
CPU: 0 PID: 32358 Comm: kworker/0:0 Tainted: G W L ------------ 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011
Workqueue: events e1000_watchdog [e1000]
task: ffff9a1dc7243150 ti: ffff9a1d25700000 task.ti: ffff9a1d25700000
RIP: 0010:[<ffffffff81783045>] [<ffffffff81783045>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0000:ffff9a1d25703d90 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff9a1d25703d58 RCX: 001a695927232e85
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff9a1d25703d90 R08: ffff9a1e3cde70b8 R09: 9786a6a47a1e70b0
R10: 9786a6a47a1e70b0 R11: ffffec91812dc500 R12: ffff9a1d25703d50
R13: 0000000000000001 R14: ffff9a1ed9c1ad00 R15: ffff9a1dc72431b8
FS: 00007f51699d4700(0000) GS:ffff9a1ed9c00000(0000) knlGS:00000000082d6840
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f51d7933580 CR3: 000000020abd0000 CR4: 00000000000407f0
Call Trace:
[<ffffffffc035660c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc03566bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff810be21f>] process_one_work+0x17f/0x440
[<ffffffff810bf336>] worker_thread+0x126/0x3c0
[<ffffffff810bf210>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff810c61f1>] kthread+0xd1/0xe0
[<ffffffff810c6120>] ? insert_kthread_work+0x40/0x40
[<ffffffff8178dd24>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffff810c6120>] ? insert_kthread_work+0x40/0x40
Code: f6 3d 99 ff 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 e8 d6 3d 99 ff 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #20221
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb8589785
Build id: 3.10.0-1160.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc045d60c
Build id: 3.10.0-1160.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc045d6bd
Build id: 3.10.0-1160.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffb7ebdc4f
Build id: 3.10.0-1160.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffb7ebed66
Build id: 3.10.0-1160.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffb7ebec40
Build id: 3.10.0-1160.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffb7ec5c21
Build id: 3.10.0-1160.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffb7ec5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb8593df7
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 30s! [kworker/4:2:144]
Modules linked in: tcp_lp nls_utf8 isofs xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore coretemp sg iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper
ablk_helper cryptd joydev pcspkr vmw_balloon parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit ata_piix libnvdimm crct10dif_pclmul crct10dif_common crc32c_intel libata mptspi scsi_transport_spi serio_raw e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 144 Comm: kworker/4:2 Kdump: loaded Tainted: G L ------------ 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
Workqueue: events e1000_watchdog [e1000]
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 30s! [kworker/7:1:61]
task: ffff8ab631948000 ti: ffff8ab63195c000 task.ti: ffff8ab63195c000
RIP: 0010:[<ffffffffb8589785>] [<ffffffffb8589785>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8ab63195fd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff8ab635713940 RCX: 000030cdb782b6a2
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8ab63195fd90 R08: ffff8ab630c230b8 R09: d526c7ad6e0230b0
R10: d526c7ad6e0230b0 R11: 0000000000000293 R12: ffff8ab63195fd50
R13: 0000000000000001 R14: ffff8ab63571ad40 R15: ffff8ab631948068
FS: 0000000000000000(0000) GS:ffff8ab635700000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000024c9a00 CR3: 0000000173ba0000 CR4: 00000000003607e0
Modules linked in: tcp_lp nls_utf8 isofs xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore coretemp sg iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper
Call Trace:
[<ffffffffc045d60c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc045d6bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffb7ebdc4f>] process_one_work+0x17f/0x440
[<ffffffffb7ebed66>] worker_thread+0x126/0x3c0
[<ffffffffb7ebec40>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffb7ec5c21>] kthread+0xd1/0xe0
[<ffffffffb7ec5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffb8593df7>] ret_from_fork_nospec_begin+0x21/0x21
Complete report #20134
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff82f89805
Build id: 3.10.0-1160.2.2.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc046560c
Build id: 3.10.0-1160.2.2.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc04656bd
Build id: 3.10.0-1160.2.2.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff828bdc4f
Build id: 3.10.0-1160.2.2.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffff828bed66
Build id: 3.10.0-1160.2.2.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff828bec40
Build id: 3.10.0-1160.2.2.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff828c5c21
Build id: 3.10.0-1160.2.2.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff828c5b50
Build id: 3.10.0-1160.2.2.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff82f93ddd
Build id: 3.10.0-1160.2.2.el7.x86_64
|
0x7
|
10
|
|
vmlinux |
0xffffffff828c5b50
Build id: 3.10.0-1160.2.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [kworker/0:2:66362]
Modules linked in: veth tcp_lp binfmt_misc xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi joydev snd_ac97_codec pcspkr ac97_bus snd_seq snd_seq_device snd_pcm snd_timer sg snd i2c_piix4 soundcore vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul
crct10dif_common crc32c_intel vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops nfit serio_raw ttm libnvdimm ata_piix e1000 drm libata mptspi scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: cvblk]
CPU: 0 PID: 66362 Comm: kworker/0:2 Kdump: loaded Tainted: P OE ------------ T 3.10.0-1160.2.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
Workqueue: events e1000_watchdog [e1000]
task: ffff9856b3173180 ti: ffff98572f4d0000 task.ti: ffff98572f4d0000
RIP: 0010:[<ffffffff82f89805>] [<ffffffff82f89805>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0000:ffff98572f4d3d90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff98572f4d3d60 RCX: 0003bd88aaea97a4
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff98572f4d3d90 R08: ffff98572f8af0b8 R09: 9ea2da2b6ccaf0b0
R10: 9ea2da2b6ccaf0b0 R11: 0000000000000000 R12: ffffffffc079d578
R13: ffffffff828ad8b3 R14: ffff98572f4d3d10 R15: ffffffffc079d578
FS: 0000000000000000(0000) GS:ffff985739600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f1864651000 CR3: 0000000011810000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc046560c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc04656bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff828bdc4f>] process_one_work+0x17f/0x440
[<ffffffff828bed66>] worker_thread+0x126/0x3c0
[<ffffffff828bec40>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff828c5c21>] kthread+0xd1/0xe0
[<ffffffff828c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff82f93ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff828c5b50>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #18396
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff816ab845
Build id: 3.10.0-693.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc00db13c
Build id: 3.10.0-693.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc00db1ed
Build id: 3.10.0-693.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff810a881a
Build id: 3.10.0-693.el7.x86_64
|
0x17a
|
5
|
worker_thread
|
vmlinux |
0xffffffff810a94e6
Build id: 3.10.0-693.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff810a93c0
Build id: 3.10.0-693.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
|
0xcf
|
8
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
9
|
ret_from_fork
|
vmlinux |
0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
|
0x58
|
10
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [kworker/0:0:20627]
Modules linked in: xt_addrtype ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_conntrack nf_conntrack_netlink iptable_filter xt_nat veth br_netfilter overlay(T) tcp_lp fuse tun ip_set nfnetlink bridge stp llc iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event sb_edac edac_core iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq pcspkr joydev snd_seq_device snd_pcm sg snd_timer snd soundcore nfit libnvdimm vmw_vmci i2c_piix4 shpchp parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi
crct10dif_pclmul crct10dif_common vmwgfx crc32c_intel drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm serio_raw drm e1000 mptspi scsi_transport_spi mptscsih mptbase ata_piix libata i2c_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: iptable_filter]
CPU: 0 PID: 20627 Comm: kworker/0:0 Tainted: G W L ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
Workqueue: events e1000_watchdog [e1000]
task: ffff88004246dee0 ti: ffff880078d60000 task.ti: ffff880078d60000
RIP: 0010:[<ffffffff816ab845>] [<ffffffff816ab845>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff880078d63d90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff880078d63d20 RCX: 000f3438a9237dba
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff880078d63d90 R08: ffff880078a2f0b8 R09: dffda574b5e2f0b0
R10: dffda574b5e2f0b0 R11: 0000000000000001 R12: ffff880035d59800
R13: ffffffff81ae4900 R14: ffff880035d59800 R15: 0000000000001400
FS: 0000000000000000(0000) GS:ffff88007b600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f15bfb84000 CR3: 000000006444b000 CR4: 00000000003407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff880078d63dd0 ffffffffc00db13c 0000ffff8158c6d4 00000000d2df3c25
ffff880078a2f0b0 ffff880078a2e8c0 ffff880078a2e000 ffff88003585ba40
ffff880078d63e18 ffffffffc00db1ed ffff88007b616540 00000000d2df3c25
Call Trace:
[<ffffffffc00db13c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc00db1ed>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff810a881a>] process_one_work+0x17a/0x440
[<ffffffff810a94e6>] worker_thread+0x126/0x3c0
[<ffffffff810a93c0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #17433
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff97f6a0f5
Build id: 3.10.0-957.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc01684dc
Build id: 3.10.0-957.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc016858d
Build id: 3.10.0-957.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff978b9d4f
Build id: 3.10.0-957.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffff978bade6
Build id: 3.10.0-957.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff978bacc0
Build id: 3.10.0-957.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff978c1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff978c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff97f74c37
Build id: 3.10.0-957.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffff978c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 42s! [kworker/0:0:84780]
Modules linked in: veth xt_nat xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock fuse vmhgfs(OE) overlay(T) sunrpc snd_seq_midi snd_seq_midi_event coretemp ppdev iosf_mbi vmw_balloon crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq joydev pcspkr snd_seq_device snd_pcm snd_timer snd soundcore sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul
crct10dif_common ata_generic pata_acpi crc32c_intel serio_raw vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix e1000 libata drm_panel_orientation_quirks mptspi scsi_transport_spi mptscsih mptbase nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 84780 Comm: kworker/0:0 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
Workqueue: events e1000_watchdog [e1000]
task: ffff885275f0b0c0 ti: ffff88527e228000 task.ti: ffff88527e228000
RIP: 0010:[<ffffffff97f6a0f5>] [<ffffffff97f6a0f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff88527e22bd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffff988e3c00 RCX: 00017d8acf4b4241
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff88527e22bd90 R08: ffff88537e0090b8 R09: deb190513b4090b0
R10: deb190513b4090b0 R11: 0000000000000000 R12: ffffffff988e3c00
R13: ffffffff98498580 R14: ffffffff988e3c00 R15: ffff885275f0b128
FS: 0000000000000000(0000) GS:ffff88537ea00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff958e2a008 CR3: 000000038aa96000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc01684dc>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc016858d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff978b9d4f>] process_one_work+0x17f/0x440
[<ffffffff978bade6>] worker_thread+0x126/0x3c0
[<ffffffff978bacc0>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffff978c1c31>] kthread+0xd1/0xe0
[<ffffffff978c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff97f74c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff978c1b60>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #13122
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff99116475
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc01fd45c
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc01fd50d
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff98ab312f
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffff98ab3df6
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff98ab3cd0
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff98abb161
Build id: 3.10.0-862.3.2.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff98abb090
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff99120677
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffff98abb090
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 27s! [kworker/1:2:81016]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm sb_edac snd_timer coretemp snd iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon soundcore vmw_vmci joydev pcspkr sg shpchp i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi
vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul drm crct10dif_common crc32c_intel ata_piix libata serio_raw mptspi scsi_transport_spi e1000 mptscsih mptbase i2c_core floppy
CPU: 1 PID: 81016 Comm: kworker/1:2 Kdump: loaded Tainted: G L ------------ 3.10.0-862.3.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/20/2014
Workqueue: events e1000_watchdog [e1000]
task: ffff9f77d5141fa0 ti: ffff9f77ce4dc000 task.ti: ffff9f77ce4dc000
RIP: 0010:[<ffffffff99116475>] [<ffffffff99116475>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9f77ce4dfd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffff98bb3c26 RCX: 00026220bdc615d0
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff9f77ce4dfd90 R08: ffff9f77f89970b8 R09: 821fbd1235d970b0
R10: 821fbd1235d970b0 R11: 0000000000000000 R12: ffff9f77fb658b00
R13: 0000000100000002 R14: ffff9f77fb698b00 R15: ffff9f77fa2cc000
FS: 0000000000000000(0000) GS:ffff9f77fb640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000f7751000 CR3: 0000000060a5a000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc01fd45c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc01fd50d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff98ab312f>] process_one_work+0x17f/0x440
[<ffffffff98ab3df6>] worker_thread+0x126/0x3c0
[<ffffffff98ab3cd0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff98abb161>] kthread+0xd1/0xe0
[<ffffffff98abb090>] ? insert_kthread_work+0x40/0x40
[<ffffffff99120677>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff98abb090>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #8813
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb136b455
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc02894dc
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc028958d
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffb0cb9ebf
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffb0cbaf56
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffb0cbae30
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffb0cc1da1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffb0cc1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb1375c37
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffffb0cc1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [kworker/0:1:6305]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink sunrpc ext4 mbcache jbd2 joydev ppdev vboxvideo(OE) nfit libnvdimm snd_intel8x0 snd_ac97_codec iosf_mbi vboxguest(OE) ttm drm_kms_helper ac97_bus crc32_pclmul syscopyarea sysfillrect ghash_clmulni_intel snd_seq sysimgblt aesni_intel snd_seq_device snd_pcm fb_sys_fops lrw gf128mul parport_pc drm snd_timer glue_helper snd ablk_helper parport video sg drm_panel_orientation_quirks cryptd soundcore pcspkr i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic
pata_acpi ahci crct10dif_pclmul crct10dif_common crc32c_intel ata_piix libahci libata serio_raw e1000 dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6305 Comm: kworker/0:1 Kdump: loaded Tainted: G W OEL ------------ 3.10.0-957.21.3.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
Workqueue: events e1000_watchdog [e1000]
task: ffff9d0c970b5140 ti: ffff9d06ba38c000 task.ti: ffff9d06ba38c000
RIP: 0010:[<ffffffffb136b455>] [<ffffffffb136b455>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9d06ba38fd90 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffffffb1ce3c00 RCX: 0001e33342e950b1
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff9d06ba38fd90 R08: ffff9d0c9dec70b8 R09: 8bcd255adb2c70b0
R10: 8bcd255adb2c70b0 R11: 0000000000000000 R12: ffffffffb1ce3c00
R13: ffffffffb18985c0 R14: ffffffffb1ce3c00 R15: ffff9d06ba38fd18
FS: 0000000000000000(0000) GS:ffff9d0c9f200000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc266ef68b0 CR3: 0000001f98c9a000 CR4: 00000000000606f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc02894dc>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc028958d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffb0cb9ebf>] process_one_work+0x17f/0x440
[<ffffffffb0cbaf56>] worker_thread+0x126/0x3c0
[<ffffffffb0cbae30>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffffb0cc1da1>] kthread+0xd1/0xe0
[<ffffffffb0cc1cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb1375c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb0cc1cd0>] ? insert_kthread_work+0x40/0x40
Code: 14 25 50 43 83 b1 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 ff 14 25 50 43 83 b1 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #1691
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb85883c5
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc018e60c
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc018e6bd
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffffb7ebe6bf
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffffb7ebf7d6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffffb7ebf6b0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffffb7ec6691
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffb7ec65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb8592d1d
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x7
|
10
|
|
vmlinux |
0xffffffffb7ec65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 34s! [kworker/0:1:12417]
Modules linked in: tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack vmhgfs(OE) ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev snd_ens1371 snd_rawmidi snd_ac97_codec vmw_balloon aesni_intel ac97_bus lrw gf128mul snd_seq glue_helper
ablk_helper cryptd snd_seq_device snd_pcm joydev pcspkr sg vmw_vmci snd_timer snd soundcore i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common syscopyarea sysfillrect ata_piix sysimgblt crc32c_intel fb_sys_fops ttm mptspi libata scsi_transport_spi drm nfit mptscsih libnvdimm serio_raw mptbase e1000 drm_panel_orientation_quirks
CPU: 0 PID: 12417 Comm: kworker/0:1 Kdump: loaded Tainted: G OEL ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
Workqueue: events e1000_watchdog [e1000]
task: ffffa03c06aa8000 ti: ffffa03c269bc000 task.ti: ffffa03c269bc000
RIP: 0010:[<ffffffffb85883c5>] [<ffffffffb85883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffffa03c269bfd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffa03c269bfd58 RCX: 000010995a81b713
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffffa03c269bfd90 R08: ffffa03d355d90b8 R09: 7f0acac6f29d90b0
R10: 7f0acac6f29d90b0 R11: 0000000000000293 R12: ffffa03c269bfd50
R13: 0000000000000001 R14: ffffa03d3961ad40 R15: ffffa03c06aa8068
FS: 0000000000000000(0000) GS:ffffa03d39600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f5b1ef20000 CR3: 0000000134b96000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc018e60c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc018e6bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffb7ebe6bf>] process_one_work+0x17f/0x440
[<ffffffffb7ebf7d6>] worker_thread+0x126/0x3c0
[<ffffffffb7ebf6b0>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffb7ec6691>] kthread+0xd1/0xe0
[<ffffffffb7ec65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb8592d1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffb7ec65c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #830
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff816ab905
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc005d13c
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc005d1ed
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff810a882a
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x17a
|
5
|
worker_thread
|
vmlinux |
0xffffffff810a94f6
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff810a93d0
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff810b099f
Build id: 3.10.0-693.5.2.el7.x86_64
|
0xcf
|
8
|
|
vmlinux |
0xffffffff810b08d0
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x40
|
9
|
ret_from_fork
|
vmlinux |
0xffffffff816b4fd8
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x58
|
10
|
|
vmlinux |
0xffffffff810b08d0
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [kworker/1:2:13145]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter xt_set ip_set nfnetlink ip_vs nf_conntrack vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event sb_edac edac_core coretemp iosf_mbi crc32_pclmul snd_ens1371 ghash_clmulni_intel snd_rawmidi aesni_intel snd_ac97_codec lrw gf128mul ac97_bus snd_seq glue_helper ablk_helper snd_seq_device ppdev cryptd snd_pcm vmw_balloon snd_timer snd joydev pcspkr soundcore vmw_vmci sg i2c_piix4 shpchp nfit libnvdimm parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs
libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi scsi_transport_spi libata crct10dif_pclmul e1000 crct10dif_common mptscsih crc32c_intel mptbase serio_raw i2c_core
CPU: 1 PID: 13145 Comm: kworker/1:2 Not tainted 3.10.0-693.5.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
Workqueue: events e1000_watchdog [e1000]
task: ffff880137133f40 ti: ffff8800a46e8000 task.ti: ffff8800a46e8000
RIP: 0010:[<ffffffff816ab905>] [<ffffffff816ab905>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8800a46ebd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 000c604b607b4c2c
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8800a46ebd90 R08: ffff880136af50b8 R09: dffaad43f3ef50b0
R10: dffaad43f3ef50b0 R11: 0000000000000000 R12: ffff880137133fa8
R13: ffffffff810ca2ae R14: ffff8800a46ebd18 R15: 0000102e2f8ce61d
FS: 0000000000000000(0000) GS:ffff880139640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 000055abf4ae4528 CR3: 00000000b405b000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8800a46ebdd0 ffffffffc005d13c 00000000b55c9038 000000005f06c3a2
ffff880136af50b0 ffff880136af48c0 ffff880136af4000 ffff8801320d0bc0
ffff8800a46ebe18 ffffffffc005d1ed ffff8800b578f080 000000005f06c3a2
Call Trace:
[<ffffffffc005d13c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc005d1ed>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff810a882a>] process_one_work+0x17a/0x440
[<ffffffff810a94f6>] worker_thread+0x126/0x3c0
[<ffffffff810a93d0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b099f>] kthread+0xcf/0xe0
[<ffffffff810b08d0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4fd8>] ret_from_fork+0x58/0x90
[<ffffffff810b08d0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #319
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff85d6a0f5
Build id: 3.10.0-957.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc041d4dc
Build id: 3.10.0-957.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc041d58d
Build id: 3.10.0-957.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff856b9d4f
Build id: 3.10.0-957.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffff856bade6
Build id: 3.10.0-957.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff856bacc0
Build id: 3.10.0-957.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff856c1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff856c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff85d74c1d
Build id: 3.10.0-957.el7.x86_64
|
0x7
|
10
|
|
vmlinux |
0xffffffff856c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kworker/2:0:41742]
Modules linked in: tcp_lp nls_utf8 isofs fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vmw_vmci vsock sunrpc snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore joydev pcspkr sg parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_pclmul crct10dif_common crc32c_intel ata_generic
pata_acpi serio_raw vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm mptspi scsi_transport_spi e1000 libata mptscsih mptbase drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 41742 Comm: kworker/2:0 Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
Workqueue: events e1000_watchdog [e1000]
task: ffff8e3470490000 ti: ffff8e3342304000 task.ti: ffff8e3342304000
RIP: 0010:[<ffffffff85d6a0f5>] [<ffffffff85d6a0f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8e3342307d90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff8e3342307d38 RCX: 0001d26bfc71f004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8e3342307d90 R08: ffff8e3475e250b8 R09: c72db6ab332250b0
R10: c72db6ab332250b0 R11: ffffe8a7c5d10c00 R12: ffff8e347869ab80
R13: ffff8e346fe8e180 R14: ffffffff856d2ed9 R15: ffff8e3342307d10
FS: 0000000000000000(0000) GS:ffff8e3478680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8efa4c93bc CR3: 0000000172faa000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc041d4dc>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc041d58d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff856b9d4f>] process_one_work+0x17f/0x440
[<ffffffff856bade6>] worker_thread+0x126/0x3c0
[<ffffffff856bacc0>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffff856c1c31>] kthread+0xd1/0xe0
[<ffffffff856c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff85d74c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff856c1b60>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #239
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff89f817c5
Build id: 3.10.0-1062.el7.x86_64
|
0x15
|
2
|
e1000_update_stats
|
e1000 |
0xffffffffc024f60c
Build id: 3.10.0-1062.el7.x86_64
|
0x94c
|
3
|
e1000_watchdog
|
e1000 |
0xffffffffc024f6bd
Build id: 3.10.0-1062.el7.x86_64
|
0x6d
|
4
|
process_one_work
|
vmlinux |
0xffffffff898bd0ff
Build id: 3.10.0-1062.el7.x86_64
|
0x17f
|
5
|
worker_thread
|
vmlinux |
0xffffffff898be216
Build id: 3.10.0-1062.el7.x86_64
|
0x126
|
6
|
|
vmlinux |
0xffffffff898be0f0
Build id: 3.10.0-1062.el7.x86_64
|
0x2a0
|
7
|
kthread
|
vmlinux |
0xffffffff898c50d1
Build id: 3.10.0-1062.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff898c5000
Build id: 3.10.0-1062.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff89f8bd37
Build id: 3.10.0-1062.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffff898c5000
Build id: 3.10.0-1062.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [kworker/0:0:28189]
Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack libcrc32c ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq vmw_balloon snd_seq_device aesni_intel lrw gf128mul snd_pcm glue_helper ablk_helper cryptd snd_timer joydev pcspkr snd sg soundcore vmw_vmci i2c_piix4 parport_pc
parport ip_tables ext4 mbcache jbd2 sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common nfit crc32c_intel ata_piix libnvdimm mptspi libata scsi_transport_spi serio_raw mptscsih e1000 mptbase drm_panel_orientation_quirks
CPU: 0 PID: 28189 Comm: kworker/0:0 Kdump: loaded Tainted: G L ------------ 3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
Workqueue: events e1000_watchdog [e1000]
task: ffff8828d4d78000 ti: ffff8828f7bf8000 task.ti: ffff8828f7bf8000
RIP: 0010:[<ffffffff89f817c5>] [<ffffffff89f817c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8828f7bfbd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffff8a901c00 RCX: 000883dcd5a39f9c
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8828f7bfbd90 R08: ffff8828f8d650b8 R09: df5ba4cfb61650b0
R10: df5ba4cfb61650b0 R11: 0000000000000001 R12: ffffffff8a901c00
R13: ffffffff8a498c00 R14: ffffffff8a901c00 R15: ffffffffc062b578
FS: 0000000000000000(0000) GS:ffff8828fb600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f903b1bf000 CR3: 00000000766da000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc024f60c>] e1000_update_stats+0x94c/0x990 [e1000]
[<ffffffffc024f6bd>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffff898bd0ff>] process_one_work+0x17f/0x440
[<ffffffff898be216>] worker_thread+0x126/0x3c0
[<ffffffff898be0f0>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff898c50d1>] kthread+0xd1/0xe0
[<ffffffff898c5000>] ? insert_kthread_work+0x40/0x40
[<ffffffff89f8bd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff898c5000>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48