Complete report #801294
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
__raw_callee_save___pv_queued_spin_unlock
|
vmlinux |
0xffffffffa9b11ae4
Build id: 3.10.0-957.el7.x86_64
|
0x10
|
2
|
igrab
|
vmlinux |
0xffffffffa9c5e29a
Build id: 3.10.0-957.el7.x86_64
|
0x4a
|
3
|
nfs_delegation_grab_inode
|
nfsv4 |
0xffffffffc061e7ba
Build id: 3.10.0-957.el7.x86_64
|
0x2a
|
4
|
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc061fb3c
Build id: 3.10.0-957.el7.x86_64
|
0xac
|
5
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc0619e48
Build id: 3.10.0-957.el7.x86_64
|
0x138
|
6
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc061bf80
Build id: 3.10.0-957.el7.x86_64
|
0x5d0
|
7
|
|
nfsv4 |
0xffffffffc061c280
Build id: 3.10.0-957.el7.x86_64
|
0x8d0
|
8
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc061c29f
Build id: 3.10.0-957.el7.x86_64
|
0x1f
|
9
|
kthread
|
vmlinux |
0xffffffffa9ac1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
10
|
|
vmlinux |
0xffffffffa9ac1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
11
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffaa174c1d
Build id: 3.10.0-957.el7.x86_64
|
0x7
|
12
|
|
vmlinux |
0xffffffffa9ac1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [18.0.16.140-man:252496]
Modules linked in: tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag binfmt_misc fuse btrfs raid6_pq xor vfat msdos fat ext4 mbcache jbd2 cdp(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 pv_channel(OE) kvm_ivshmem(OE) dns_resolver nfs lockd grace fscache sunrpc ppdev nfit libnvdimm iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev sg virtio_balloon pcspkr i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom virtio_net virtio_blk virtio_console ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix drm_panel_orientation_quirks libata serio_raw virtio_pci virtio_ring virtio floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 252496 Comm: 18.0.16.140-man Kdump: loaded Tainted: G OE ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.10.2-0-g5f4c7b1-20181220_000000-szxrtosci10000 04/01/2014
task: ffff97e457c95140 ti: ffff97e45831c000 task.ti: ffff97e45831c000
RIP: 0010:[<ffffffffa9b11ae4>] [<ffffffffa9b11ae4>] __raw_callee_save___pv_queued_spin_unlock+0x10/0x17
RSP: 0018:ffff97e45831fd60 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff97e3fd2572e0 RCX: ffff97e44cf0d000
RDX: 0000000000000000 RSI: 0000000000000005 RDI: ffff97e0894e5c60
RBP: ffff97e45831fd68 R08: ffff97e44cf0e08c R09: 000000018040001d
R10: 00000000fd254e01 R11: ffffed749ef49400 R12: ffff97e45831fd00
R13: 000000018040001d R14: 0000000000000286 R15: 0000000000000000
FS: 0000000000000000(0000) GS:ffff97e45b600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f21cc137ac0 CR3: 000000080af50000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa9c5e29a>] igrab+0x4a/0x60
[<ffffffffc061e7ba>] nfs_delegation_grab_inode+0x2a/0x50 [nfsv4]
[<ffffffffc061fb3c>] nfs_delegation_reap_unclaimed+0xac/0x140 [nfsv4]
[<ffffffffc0619e48>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc061bf80>] nfs4_state_manager+0x5d0/0x8d0 [nfsv4]
[<ffffffffc061c280>] ? nfs4_state_manager+0x8d0/0x8d0 [nfsv4]
[<ffffffffc061c29f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffffa9ac1c31>] kthread+0xd1/0xe0
[<ffffffffa9ac1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffffaa174c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa9ac1b60>] ? insert_kthread_work+0x40/0x40
Code: 50 41 51 41 52 41 53 e8 1b 07 00 00 41 5b 41 5a 41 59 41 58 5f 5e 5a 59 5d c3 90 55 48 89 e5 52 b8 01 00 00 00 31 d2 f0 0f b0 17 <3c> 01 75 03 5a 5d c3 56 0f b6 f0 e8 bc ff ff ff 5e 5a 5d c3 0f
Complete report #13264
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
__raw_callee_save___pv_queued_spin_unlock_slowpath
|
vmlinux |
0xffffffff810fbc04
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x24
|
2
|
|
vmlinux |
0xffffffff8121f2ea
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x4a
|
3
|
nfs_delegation_grab_inode
|
nfsv4 |
0xffffffffc04d640a
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x2a
|
4
|
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc04d76fc
Build id: 3.10.0-693.17.1.el7.x86_64
|
0xac
|
5
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc04d1e68
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x138
|
6
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc04d3c08
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x5c8
|
7
|
|
nfsv4 |
0xffffffffc04d3f00
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x8c0
|
8
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc04d3f1f
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x1f
|
9
|
kthread
|
vmlinux |
0xffffffff810b270f
Build id: 3.10.0-693.17.1.el7.x86_64
|
0xcf
|
10
|
|
vmlinux |
0xffffffff810b2640
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x40
|
11
|
ret_from_fork
|
vmlinux |
0xffffffff816b8798
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x58
|
12
|
|
vmlinux |
0xffffffff810b2640
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [10.1.6.43-manag:40153]
Modules linked in: iptable_filter nfsv3 binfmt_misc rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache dm_mirror dm_region_hash dm_log dm_mod crc32_pclmul ghash_clmulni_intel virtio_balloon virtio_scsi aesni_intel virtio_net bochs_drm lrw gf128mul glue_helper ablk_helper ttm cryptd drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm joydev ppdev i2c_piix4 sg i2c_core virtio_pci virtio_ring virtio pcspkr parport_pc parport nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel ata_piix libata serio_raw floppy
CPU: 1 PID: 40153 Comm: 10.1.6.43-manag Not tainted 3.10.0-693.17.1.el7.x86_64 #1
Hardware name: Nutanix AHV, BIOS 1.9.1-5.el6 04/01/2014
task: ffff8801ad2c0000 ti: ffff880334a4c000 task.ti: ffff880334a4c000
RIP: 0010:[<ffffffff810fbc04>] [<ffffffff810fbc04>] __raw_callee_save___pv_queued_spin_unlock_slowpath+0x24/0x24
RSP: 0018:ffff880334a4fd70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000000000673c401 RCX: ffff880428e5d000
RDX: 0000000000000001 RSI: 0000000000000005 RDI: ffff8803de9afd20
RBP: ffff880334a4fd88 R08: ffff880428e5c88c R09: 000000018040003d
R10: 000000000673c401 R11: ffffea001019ce00 R12: ffff88040673d0e0
R13: ffff880334a4fd68 R14: ffff880334a4fd00 R15: 000000018040003d
FS: 0000000000000000(0000) GS:ffff88042ba40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f63de89733a CR3: 00000000019fa000 CR4: 00000000003606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8121f2ea>] ? igrab+0x4a/0x60
[<ffffffffc04d640a>] nfs_delegation_grab_inode+0x2a/0x50 [nfsv4]
[<ffffffffc04d76fc>] nfs_delegation_reap_unclaimed+0xac/0x140 [nfsv4]
[<ffffffffc04d1e68>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc04d3c08>] nfs4_state_manager+0x5c8/0x8c0 [nfsv4]
[<ffffffffc04d3f00>] ? nfs4_state_manager+0x8c0/0x8c0 [nfsv4]
[<ffffffffc04d3f1f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffff810b270f>] kthread+0xcf/0xe0
[<ffffffff810b2640>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b8798>] ret_from_fork+0x58/0x90
[<ffffffff810b2640>] ? insert_kthread_work+0x40/0x40
Code: 92 c0 5d c3 0f 1f 00 55 48 89 e5 51 52 56 57 41 50 41 51 41 52 41 53 e8 eb 06 00 00 41 5b 41 5a 41 59 41 58 5f 5e 5a 59 5d c3 90 <55> 48 89 e5 52 b8 01 00 00 00 31 d2 f0 0f b0 17 3c 01 75 03 5a
Complete report #1119477
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
nfs_sb_active
|
nfs |
0xffffffffc06f2141
Build id: 3.10.0-957.el7.x86_64
|
0x51
|
2
|
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc08e0b30
Build id: 3.10.0-957.el7.x86_64
|
0xa0
|
3
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc08dae48
Build id: 3.10.0-957.el7.x86_64
|
0x138
|
4
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc08dcf80
Build id: 3.10.0-957.el7.x86_64
|
0x5d0
|
5
|
|
nfsv4 |
0xffffffffc08dd280
Build id: 3.10.0-957.el7.x86_64
|
0x8d0
|
6
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc08dd29f
Build id: 3.10.0-957.el7.x86_64
|
0x1f
|
7
|
kthread
|
vmlinux |
0xffffffff8dcc1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff8dcc1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8e374c1d
Build id: 3.10.0-957.el7.x86_64
|
0x7
|
10
|
|
vmlinux |
0xffffffff8dcc1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#21 stuck for 23s! [222.31.135.63-m:137101]
Modules linked in: nfsd nfs_acl binfmt_misc ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) nfnetlink_queue nfnetlink_log nfnetlink bluetooth rfkill tcp_diag inet_diag rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache sunrpc iTCO_wdt iTCO_vendor_support skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg mei_me lpc_ich mei i2c_i801 wmi ipmi_si ipmi_devintf ipmi_msghandler acpi_cpufreq acpi_pad acpi_power_meter ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ast drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ttm crc32c_intel i40e igb drm ahci ptp libahci pps_core dca i2c_algo_bit libata drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 21 PID: 137101 Comm: 222.31.135.63-m Kdump: loaded Tainted: G ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: SuperCloud R5210 G11/X11DPi-N, BIOS 3.5 06/29/2021
task: ffff94f2f57e8000 ti: ffff94fa9adc4000 task.ti: ffff94fa9adc4000
RIP: 0010:[<ffffffffc06f2141>] [<ffffffffc06f2141>] nfs_sb_active+0x51/0x80 [nfs]
RSP: 0018:ffff94fa9adc7da8 EFLAGS: 00000202
RAX: 0000000000000001 RBX: ffff94f29e8df160 RCX: ffff94f287adf800
RDX: 0000000000000004 RSI: 0000000000000004 RDI: ffff94f287ade000
RBP: ffff94fa9adc7da8 R08: ffff94f287ade08c R09: ffffffffc08c258a
R10: ffff94f29e8df160 R11: ffffe60fc498be00 R12: ffff94f287ade08c
R13: ffff94fa9adc7da8 R14: ffff94f29e8df160 R15: ffffe60fc498be00
FS: 0000000000000000(0000) GS:ffff94f29e8c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007faf95211ba0 CR3: 000000015d810000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 00000000
Call Trace:
[<ffffffffc08e0b30>] nfs_delegation_reap_unclaimed+0xa0/0x140 [nfsv4]
[<ffffffffc08dae48>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc08dcf80>] nfs4_state_manager+0x5d0/0x8d0 [nfsv4]
[<ffffffffc08dd280>] ? nfs4_state_manager+0x8d0/0x8d0 [nfsv4]
[<ffffffffc08dd29f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffff8dcc1c31>] kthread+0xd1/0xe0
[<ffffffff8dcc1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff8e374c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff8dcc1b60>] ? insert_kthread_work+0x40/0x40
Code: f0 0f b1 b7 8c 00 00 00 39 d0 89 c6 75 2b b8 01 00 00 00 f0 0f c1 81 88 05 00 00 85 c0 74 13 f0 ff 8f 8c 00 00 00 b8 01 00 00 00 <5d> c3 0f 1f 44 00 00 b8 01 00 00 00 5d c3 85 f6 74 12 8d 56 01
Complete report #58114
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
nfs_sb_active
|
nfs |
0xffffffffc1f8a0be
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x2e
|
2
|
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc1fe1d20
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xa0
|
3
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc1fdc038
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x138
|
4
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc1fde170
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x5d0
|
5
|
|
nfsv4 |
0xffffffffc1fde470
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x8d0
|
6
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc1fde48f
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x1f
|
7
|
kthread
|
vmlinux |
0xffffffff8a8bdf21
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffff8a8bde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8af255f7
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffff8a8bde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [129.112.5.24-ma:20259]
Modules linked in: nvidia_uvm(OE) vfat fat uas usb_storage tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) nvidia_modeset(POE) nvidia(POE) iTCO_wdt iTCO_vendor_support eeepc_wmi asus_wmi sparse_keymap
rfkill video mxm_wmi intel_powerclamp coretemp intel_rapl iosf_mbi kvm raid456 async_raid6_recov async_memcpy irqbypass async_pq crc32_pclmul ghash_clmulni_intel snd_hda_codec_hdmi aesni_intel lrw gf128mul glue_helper ablk_helper cryptd raid0 raid6_pq libcrc32c async_xor snd_hda_codec_realtek xor async_tx snd_hda_codec_generic pcspkr drm_kms_helper snd_hda_intel syscopyarea sysfillrect snd_hda_codec sysimgblt fb_sys_fops snd_hda_core drm snd_hwdep snd_seq lpc_ich snd_seq_device i2c_i801 snd_pcm ipmi_devintf ipmi_msghandler sg snd_timer mei_me snd mei soundcore shpchp wmi ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic igb crct10dif_pclmul serio_raw ahci crct10dif_common crc32c_intel i2c_algo_bit dca libahci i2c_core e1000e libata ptp pps_core
CPU: 5 PID: 20259 Comm: 129.112.5.24-ma Kdump: loaded Tainted: P OE ------------ 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: ASUS All Series/X99-E WS/USB 3.1, BIOS 3201 06/17/2016
task: ffff8bc70e740000 ti: ffff8bc6d72a4000 task.ti: ffff8bc6d72a4000
RIP: 0010:[<ffffffffc1f8a0be>] [<ffffffffc1f8a0be>] nfs_sb_active+0x2e/0x80 [nfs]
RSP: 0018:ffff8bc6d72a7da8 EFLAGS: 00000246
RAX: 000000000000000a RBX: ffff8bc6fc396000 RCX: ffff8bc6fc396000
RDX: 000000000000000a RSI: 000000000000000b RDI: ffff8bc6fc393800
RBP: ffff8bc6d72a7da8 R08: ffff8bc6fc39388c R09: 0000000180200017
R10: 0000000004e92601 R11: fffff7773f13a400 R12: ffff8bc6fc39388c
R13: 0000000180200017 R14: 0000000004e92601 R15: fffff7773f13a400
FS: 0000000000000000(0000) GS:ffff8bc75f340000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000056450d0dcb98 CR3: 0000001588a0e000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc1fe1d20>] nfs_delegation_reap_unclaimed+0xa0/0x140 [nfsv4]
[<ffffffffc1fdc038>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc1fde170>] nfs4_state_manager+0x5d0/0x8d0 [nfsv4]
[<ffffffffc1fde470>] ? nfs4_state_manager+0x8d0/0x8d0 [nfsv4]
[<ffffffffc1fde48f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffff8a8bdf21>] kthread+0xd1/0xe0
[<ffffffff8a8bde50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8af255f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8a8bde50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 55 8b 97 8c 00 00 00 48 8b 8f 50 03 00 00 48 89 e5 85 d2 74 5b 8d 72 01 4c 8d 87 8c 00 00 00 89 d0 f0 0f b1 b7 8c 00 00 00 <39> d0 89 c6 75 2b b8 01 00 00 00 f0 0f c1 81 88 05 00 00 85 c0
Complete report #848191
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
nfs_sb_deactive
|
nfs |
0xffffffffc04a0f53
Build id: 3.10.0-693.el7.x86_64
|
0x13
|
2
|
?
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc04fa715
Build id: 3.10.0-693.el7.x86_64
|
0x115
|
3
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc04f4e58
Build id: 3.10.0-693.el7.x86_64
|
0x138
|
4
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc04f6bf8
Build id: 3.10.0-693.el7.x86_64
|
0x5c8
|
5
|
|
nfsv4 |
0xffffffffc04f6ef0
Build id: 3.10.0-693.el7.x86_64
|
0x8c0
|
6
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc04f6f0f
Build id: 3.10.0-693.el7.x86_64
|
0x1f
|
7
|
kthread
|
vmlinux |
0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
|
0xcf
|
8
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
9
|
ret_from_fork
|
vmlinux |
0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
|
0x58
|
10
|
|
vmlinux |
0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [10.34.218.196-m:11460]
Modules linked in: rpcsec_gss_krb5 auth_rpcgss binfmt_misc nfsv4 dns_resolver nfs lockd grace fscache vmw_vsock_vmci_transport vsock sunrpc edac_core iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon pcspkr joydev sg i2c_piix4 vmw_vmci shpchp nfit libnvdimm parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod vmwgfx crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul drm crct10dif_common crc32c_intel ahci serio_raw libahci ata_piix libata vmw_pvscsi vmxnet3 i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 11460 Comm: 10.34.218.196-m Not tainted 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff880fe6e23f40 ti: ffff8805c77d4000 task.ti: ffff8805c77d4000
RIP: 0010:[<ffffffffc04a0f53>] [<ffffffffc04a0f53>] nfs_sb_deactive+0x13/0x30 [nfs]
RSP: 0018:ffff8805c77d7db0 EFLAGS: 00000246
RAX: ffff880ffb5c1000 RBX: ffff88103fd19c60 RCX: ffff880ffb5c1000
RDX: 0000000000000000 RSI: 0000000000000006 RDI: ffff880ffb5c4000
RBP: ffff8805c77d7df0 R08: ffff880ffb5c408c R09: ffffffffc04f469a
R10: ffff88103fd19c60 R11: ffffea003fbc8700 R12: ffff880ffb5c1000
R13: ffff8805c77d7da8 R14: ffff880ffb5c408c R15: ffffffffc04f469a
FS: 0000000000000000(0000) GS:ffff88103fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2255b731a0 CR3: 00000000019f2000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffffffc04fa715 ffff8805807e6280 000000000c336722 0000000000000000
ffff880035d880c0 ffff880ff1af2000 ffff880ff1af2130 0000000000000000
ffff8805c77d7e58 ffffffffc04f4e58 ffff880ff1af2000 ffff880ff1af20c8
Call Trace:
[<ffffffffc04fa715>] ? nfs_delegation_reap_unclaimed+0x115/0x140 [nfsv4]
[<ffffffffc04f4e58>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc04f6bf8>] nfs4_state_manager+0x5c8/0x8c0 [nfsv4]
[<ffffffffc04f6ef0>] ? nfs4_state_manager+0x8c0/0x8c0 [nfsv4]
[<ffffffffc04f6f0f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 0f 83 c0 01 39 c8 75 f0 31 c0 5d c3 0f 1f 40 00 b8 01 00 00 00 5d c3 90 0f 1f 44 00 00 48 8b 87 50 03 00 00 f0 ff 88 88 05 00 00 <0f> 94 c2 84 d2 75 06 c3 0f 1f 44 00 00 55 48 89 e5 e8 57 36 d6
Complete report #168094
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
deactivate_super
|
vmlinux |
0xffffffffa3422b7e
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x2e
|
2
|
nfs_sb_deactive
|
nfs |
0xffffffffc1ecd179
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x29
|
3
|
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc1f4cd95
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x115
|
4
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc1f47038
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x138
|
5
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc1f49170
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x5d0
|
6
|
|
nfsv4 |
0xffffffffc1f49470
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x8d0
|
7
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc1f4948f
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x1f
|
8
|
kthread
|
vmlinux |
0xffffffffa32bdf21
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xd1
|
9
|
|
vmlinux |
0xffffffffa32bde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
10
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa39255f7
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x21
|
11
|
|
vmlinux |
0xffffffffa32bde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [129.112.5.24-ma:28711]
Modules linked in: vfat fat ses enclosure scsi_transport_sas nvidia_uvm(OE) tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) nvidia_modeset(POE) nvidia(POE) eeepc_wmi asus_wmi iTCO_wdt sparse_keymap
iTCO_vendor_support rfkill mxm_wmi video intel_powerclamp coretemp intel_rapl iosf_mbi raid456 kvm async_raid6_recov async_memcpy async_pq irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_hda_codec_hdmi raid0 raid6_pq libcrc32c async_xor xor async_tx uas snd_hda_codec_realtek pcspkr snd_hda_codec_generic usb_storage drm_kms_helper i2c_i801 snd_hda_intel syscopyarea sysfillrect sysimgblt fb_sys_fops snd_hda_codec drm snd_seq snd_hda_core snd_seq_device snd_hwdep snd_pcm ipmi_devintf ipmi_msghandler sg snd_timer mei_me snd lpc_ich mei soundcore shpchp wmi ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic igb crct10dif_pclmul i2c_algo_bit crct10dif_common dca i2c_core crc32c_intel ahci serio_raw e1000e libahci libata ptp
pps_core
CPU: 10 PID: 28711 Comm: 129.112.5.24-ma Kdump: loaded Tainted: P OE ------------ 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: ASUS All Series/X99-E WS/USB 3.1, BIOS 3201 06/17/2016
task: ffff8d9a85660000 ti: ffff8d9a75f64000 task.ti: ffff8d9a75f64000
RIP: 0010:[<ffffffffa3422b7e>] [<ffffffffa3422b7e>] deactivate_super+0x2e/0x60
RSP: 0018:ffff8d9a75f67d90 EFLAGS: 00000246
RAX: 0000000000000009 RBX: ffff8d9a9874b08c RCX: 0000000000000008
RDX: 0000000000000009 RSI: ffff8d9a9874b08c RDI: ffff8d9a9874b000
RBP: ffff8d9a75f67d98 R08: ffff8d9a9874b08c R09: 0000000100200006
R10: 00000000ef690401 R11: fffff2f679bda400 R12: 00000000ef690401
R13: fffff2f679bda400 R14: ffff8d98ef690a00 R15: ffff8d9a75f67d88
FS: 0000000000000000(0000) GS:ffff8d9a9f480000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f829038e000 CR3: 000000177d40e000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc1ecd179>] nfs_sb_deactive+0x29/0x30 [nfs]
[<ffffffffc1f4cd95>] nfs_delegation_reap_unclaimed+0x115/0x140 [nfsv4]
[<ffffffffc1f47038>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc1f49170>] nfs4_state_manager+0x5d0/0x8d0 [nfsv4]
[<ffffffffc1f49470>] ? nfs4_state_manager+0x8d0/0x8d0 [nfsv4]
[<ffffffffc1f4948f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffffa32bdf21>] kthread+0xd1/0xe0
[<ffffffffa32bde50>] ? insert_kthread_work+0x40/0x40
[<ffffffffa39255f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa32bde50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 55 48 89 e5 53 8b 97 8c 00 00 00 48 89 fb 83 fa 01 74 1d 8d 4a ff 48 8d b7 8c 00 00 00 89 d0 f0 0f b1 8f 8c 00 00 00 39 c2 <89> c1 75 16 5b 5d c3 48 8d 7b 68 e8 f2 54 4f 00 48 89 df e8 2a
Complete report #24560
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
deactivate_super
|
vmlinux |
0xffffffff85a44cdc
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x2c
|
2
|
nfs_sb_deactive
|
nfs |
0xffffffffc0b221d9
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x29
|
3
|
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc0c23ba5
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x115
|
4
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc0c1de48
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x138
|
5
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc0c1ff80
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x5d0
|
6
|
|
nfsv4 |
0xffffffffc0c20280
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x8d0
|
7
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc0c2029f
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x1f
|
8
|
kthread
|
vmlinux |
0xffffffff858c1c31
Build id: 3.10.0-957.1.3.el7.x86_64
|
0xd1
|
9
|
|
vmlinux |
0xffffffff858c1b60
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x40
|
10
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff85f74c37
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x21
|
11
|
|
vmlinux |
0xffffffff858c1b60
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [10.0.1.18-manag:26532]
Modules linked in: fuse xt_nat xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs binfmt_misc lockd grace fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc ext4 mbcache jbd2 snd_hda_codec_hdmi snd_hda_codec_realtek sb_edac intel_powerclamp coretemp snd_hda_codec_generic snd_hda_intel intel_rapl iosf_mbi kvm_intel snd_hda_codec
kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul snd_hda_core glue_helper ablk_helper cryptd snd_hwdep snd_seq ixgbe mei_wdt iTCO_wdt iTCO_vendor_support dcdbas snd_seq_device mdio dca snd_pcm sg snd_timer snd soundcore i2c_i801 pcspkr mei_me mei lpc_ich ip_tables xfs libcrc32c sd_mod sr_mod crc_t10dif crct10dif_generic cdrom nouveau video mxm_wmi wmi i2c_algo_bit drm_kms_helper crct10dif_pclmul ahci crct10dif_common crc32c_intel serio_raw syscopyarea sysfillrect sysimgblt fb_sys_fops libahci ttm e1000e ata_generic pata_acpi drm libata ptp pps_core uas drm_panel_orientation_quirks usb_storage dm_mirror dm_region_hash dm_log dm_mod
CPU: 13 PID: 26532 Comm: 10.0.1.18-manag Kdump: loaded Tainted: G L ------------ 3.10.0-957.1.3.el7.x86_64 #1
Hardware name: Dell Inc. Precision T5600/0V6XGW, BIOS A03 06/06/2012
task: ffff9babdbae4100 ti: ffff9bafa4874000 task.ti: ffff9bafa4874000
RIP: 0010:[<ffffffff85a44cdc>] [<ffffffff85a44cdc>] deactivate_super+0x2c/0x60
RSP: 0018:ffff9bafa4877d90 EFLAGS: 00000246
RAX: 0000000000000008 RBX: ffff9bb37b73c88c RCX: 0000000000000007
RDX: 0000000000000008 RSI: ffff9bb37b73c88c RDI: ffff9bb37b73c800
RBP: ffff9bafa4877d98 R08: ffff9bb37b73c88c R09: 0000000180200007
R10: 0000000077bee601 R11: ffffd66d20defb00 R12: 0000000077bee601
R13: ffffd66d20defb00 R14: ffff9bb377bef800 R15: ffff9bafa4877d88
FS: 0000000000000000(0000) GS:ffff9baf7fbc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f20c5725ac0 CR3: 0000000281c10000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc0b221d9>] nfs_sb_deactive+0x29/0x30 [nfs]
[<ffffffffc0c23ba5>] nfs_delegation_reap_unclaimed+0x115/0x140 [nfsv4]
[<ffffffffc0c1de48>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc0c1ff80>] nfs4_state_manager+0x5d0/0x8d0 [nfsv4]
[<ffffffffc0c20280>] ? nfs4_state_manager+0x8d0/0x8d0 [nfsv4]
[<ffffffffc0c2029f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffff858c1c31>] kthread+0xd1/0xe0
[<ffffffff858c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff85f74c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff858c1b60>] ? insert_kthread_work+0x40/0x40
Code: 66 66 66 90 55 48 89 e5 53 8b 97 8c 00 00 00 48 89 fb 83 fa 01 74 1d 8d 4a ff 48 8d b7 8c 00 00 00 89 d0 f0 0f b1 8f 8c 00 00 00 <39> c2 89 c1 75 16 5b 5d c3 48 8d 7b 68 e8 32 22 52 00 48 89 df
Complete report #20383
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
nfs_sb_deactive
|
nfs |
0xffffffffc2010155
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x5
|
2
|
?
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc2069d95
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x115
|
3
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc2064038
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x138
|
4
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc2066170
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x5d0
|
5
|
|
nfsv4 |
0xffffffffc2066470
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x8d0
|
6
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc206648f
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x1f
|
7
|
kthread
|
vmlinux |
0xffffffffae0bdf21
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xd1
|
8
|
|
vmlinux |
0xffffffffae0bde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffae7255f7
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x21
|
10
|
|
vmlinux |
0xffffffffae0bde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [129.112.5.24-ma:6262]
Modules linked in: tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) nvidia(POE) eeepc_wmi asus_wmi sparse_keymap iTCO_wdt rfkill iTCO_vendor_support video mxm_wmi
intel_powerclamp coretemp intel_rapl iosf_mbi raid456 kvm async_raid6_recov async_memcpy async_pq irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper snd_hda_codec_hdmi cryptd raid0 raid6_pq libcrc32c async_xor snd_hda_codec_realtek xor snd_hda_codec_generic pcspkr async_tx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops snd_hda_intel i2c_i801 snd_hda_codec drm snd_seq snd_hda_core snd_hwdep snd_seq_device ipmi_devintf snd_pcm ipmi_msghandler sg snd_timer mei_me snd mei lpc_ich soundcore shpchp wmi ip_tables ext4 mbcache jbd2 uas usb_storage sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel igb serio_raw ahci libahci e1000e i2c_algo_bit dca i2c_core libata ptp pps_core
CPU: 5 PID: 6262 Comm: 129.112.5.24-ma Kdump: loaded Tainted: P OE ------------ 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: ASUS All Series/X99-E WS/USB 3.1, BIOS 3201 06/17/2016
task: ffff9484d22cbf40 ti: ffff9484b9ac8000 task.ti: ffff9484b9ac8000
RIP: 0010:[<ffffffffc2010155>] [<ffffffffc2010155>] nfs_sb_deactive+0x5/0x30 [nfs]
RSP: 0018:ffff9484b9acbdb0 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000009 RCX: ffff9484d174c800
RDX: 0000000000000000 RSI: 0000000000000009 RDI: ffff9484d174b000
RBP: ffff9484b9acbdf0 R08: ffff9484d174b08c R09: 0000000180200017
R10: 00000000dbbec001 R11: ffffdfae206efb00 R12: ffff9484d174c800
R13: 0000000000000000 R14: ffff9484d174b08c R15: 0000000180200017
FS: 0000000000000000(0000) GS:ffff9484df340000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000556493195610 CR3: 000000119e40e000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc2069d95>] ? nfs_delegation_reap_unclaimed+0x115/0x140 [nfsv4]
[<ffffffffc2064038>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc2066170>] nfs4_state_manager+0x5d0/0x8d0 [nfsv4]
[<ffffffffc2066470>] ? nfs4_state_manager+0x8d0/0x8d0 [nfsv4]
[<ffffffffc206648f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffffae0bdf21>] kthread+0xd1/0xe0
[<ffffffffae0bde50>] ? insert_kthread_work+0x40/0x40
[<ffffffffae7255f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffae0bde50>] ? insert_kthread_work+0x40/0x40
Code: c0 eb 0c 0f 1f 00 48 63 d0 39 74 97 04 74 0f 83 c0 01 39 c8 75 f0 31 c0 5d c3 0f 1f 40 00 b8 01 00 00 00 5d c3 90 0f 1f 44 00 00 <48> 8b 87 50 03 00 00 f0 ff 88 88 05 00 00 0f 94 c2 84 d2 75 06
Complete report #486109
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_lock
|
vmlinux |
0xffffffff816abc00
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x10
|
2
|
?
nfs_delegation_grab_inode
|
nfsv4 |
0xffffffffc05e13fb
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x1b
|
3
|
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc05e26fc
Build id: 3.10.0-693.5.2.el7.x86_64
|
0xac
|
4
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc05dce68
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x138
|
5
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc05dec08
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x5c8
|
6
|
|
nfsv4 |
0xffffffffc05def00
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x8c0
|
7
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc05def1f
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x1f
|
8
|
kthread
|
vmlinux |
0xffffffff810b099f
Build id: 3.10.0-693.5.2.el7.x86_64
|
0xcf
|
9
|
|
vmlinux |
0xffffffff810b08d0
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x40
|
10
|
ret_from_fork
|
vmlinux |
0xffffffff816b4fd8
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x58
|
11
|
|
vmlinux |
0xffffffff810b08d0
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [20.0.0.11-manag:166576]
Modules linked in: xfs libcrc32c rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache 8021q garp mrp stp llc ip6_tables iptable_filter sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg joydev iTCO_wdt pcspkr mxm_wmi iTCO_vendor_support ipmi_si dcdbas mei_me mei ipmi_devintf ipmi_msghandler lpc_ich shpchp wmi acpi_power_meter tpm_crb nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci libahci tg3 crct10dif_pclmul crct10dif_common crc32c_intel libata megaraid_sas ptp i2c_core pps_core
CPU: 10 PID: 166576 Comm: 20.0.0.11-manag Not tainted 3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R530/0CN7X8, BIOS 2.11.0 12/23/2019
task: ffff880075b35ee0 ti: ffff88045b3d8000 task.ti: ffff88045b3d8000
RIP: 0010:[<ffffffff816abc00>] [<ffffffff816abc00>] _raw_spin_lock+0x10/0x30
RSP: 0018:ffff88045b3dbd90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff88017936188c RCX: ffff880179363800
RDX: 0000000000000001 RSI: 0000000000000004 RDI: ffff88084efe8ed0
RBP: ffff88045b3dbda8 R08: ffff88017936188c R09: ffffffffc05c54ba
R10: ffff88046e759c80 R11: ffffea001c4e0800 R12: ffff88046e759c80
R13: ffffea001c4e0800 R14: 0000000013820000 R15: ffff88045b3dbd88
FS: 0000000000000000(0000) GS:ffff88046e740000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f97d1f004f0 CR3: 00000000019f2000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffffffc05e13fb ffff88084efe8e80 ffff88046d331cc8 ffff88045b3dbdf0
ffffffffc05e26fc ffff88084efe8e80 000000009827a864 0000000000000000
ffff88086944ba40 ffff88046d331c00 ffff88086944ba50 ffff88086944ba50
Call Trace:
[<ffffffffc05e13fb>] ? nfs_delegation_grab_inode+0x1b/0x50 [nfsv4]
[<ffffffffc05e26fc>] nfs_delegation_reap_unclaimed+0xac/0x140 [nfsv4]
[<ffffffffc05dce68>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc05dec08>] nfs4_state_manager+0x5c8/0x8c0 [nfsv4]
[<ffffffffc05def00>] ? nfs4_state_manager+0x8c0/0x8c0 [nfsv4]
[<ffffffffc05def1f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffff810b099f>] kthread+0xcf/0xe0
[<ffffffff810b08d0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4fd8>] ret_from_fork+0x58/0x90
[<ffffffff810b08d0>] ? insert_kthread_work+0x40/0x40
Code: 44 00 00 85 d2 74 e4 0f 1f 40 00 eb ed 66 0f 1f 44 00 00 b8 01 00 00 00 5d c3 90 0f 1f 44 00 00 31 c0 ba 01 00 00 00 f0 0f b1 17 <85> c0 75 01 c3 55 89 c6 48 89 e5 e8 a4 2a ff ff 5d c3 0f 1f 40
Complete report #230336
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_lock
|
vmlinux |
0xffffffff9211b744
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x14
|
2
|
|
vmlinux |
0xffffffff91c3bf3e
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x1e
|
3
|
nfs_delegation_grab_inode
|
nfsv4 |
0xffffffffc2ec69aa
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x2a
|
4
|
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc2ec7d2c
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xac
|
5
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc2ec2038
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x138
|
6
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc2ec4170
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x5d0
|
7
|
|
nfsv4 |
0xffffffffc2ec4470
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x8d0
|
8
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc2ec448f
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x1f
|
9
|
kthread
|
vmlinux |
0xffffffff91abdf21
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xd1
|
10
|
|
vmlinux |
0xffffffff91abde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
11
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff921255f7
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x21
|
12
|
|
vmlinux |
0xffffffff91abde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [129.112.5.24-ma:32034]
Modules linked in: uas usb_storage tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) nvidia(POE) iTCO_wdt iTCO_vendor_support eeepc_wmi asus_wmi sparse_keymap
rfkill video mxm_wmi intel_powerclamp coretemp intel_rapl iosf_mbi kvm raid456 async_raid6_recov async_memcpy async_pq irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel snd_hda_codec_hdmi lrw gf128mul glue_helper ablk_helper cryptd raid0 raid6_pq libcrc32c async_xor pcspkr xor snd_hda_codec_realtek async_tx snd_hda_codec_generic drm_kms_helper i2c_i801 syscopyarea sysfillrect sysimgblt snd_hda_intel fb_sys_fops drm snd_hda_codec snd_seq snd_hda_core snd_seq_device snd_hwdep snd_pcm sg snd_timer snd soundcore mei_me mei lpc_ich shpchp wmi ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common serio_raw igb crc32c_intel ahci libahci e1000e i2c_algo_bit dca i2c_core libata ptp pps_core
CPU: 11 PID: 32034 Comm: 129.112.5.24-ma Kdump: loaded Tainted: P OE ------------ 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: ASUS All Series/X99-E WS/USB 3.1, BIOS 3201 06/17/2016
task: ffff9cdbcaecaf70 ti: ffff9cdbaf704000 task.ti: ffff9cdbaf704000
RIP: 0010:[<ffffffff9211b744>] [<ffffffff9211b744>] _raw_spin_lock+0x14/0x30
RSP: 0018:ffff9cdbaf707d70 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff9cdb2c29bc00 RCX: ffff9cdbcaa9c000
RDX: 0000000000000001 RSI: 0000000000000009 RDI: ffff9cdbfbc30e98
RBP: ffff9cdbaf707d88 R08: ffff9cdbcaa9b08c R09: 0000000180200013
R10: 000000002c29a201 R11: ffffe713fcb0a600 R12: ffff9cdb2c29bce0
R13: ffffffff91aa64cb R14: ffff9cdbaf707d00 R15: 0000000180200013
FS: 0000000000000000(0000) GS:ffff9cdc1f4c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe8d7aac1a0 CR3: 0000001a3d80e000 CR4: 00000000001607e0
Call Trace:
[<ffffffff91c3bf3e>] ? igrab+0x1e/0x60
[<ffffffffc2ec69aa>] nfs_delegation_grab_inode+0x2a/0x50 [nfsv4]
[<ffffffffc2ec7d2c>] nfs_delegation_reap_unclaimed+0xac/0x140 [nfsv4]
[<ffffffffc2ec2038>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc2ec4170>] nfs4_state_manager+0x5d0/0x8d0 [nfsv4]
[<ffffffffc2ec4470>] ? nfs4_state_manager+0x8d0/0x8d0 [nfsv4]
[<ffffffffc2ec448f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffff91abdf21>] kthread+0xd1/0xe0
[<ffffffff91abde50>] ? insert_kthread_work+0x40/0x40
[<ffffffff921255f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff91abde50>] ? insert_kthread_work+0x40/0x40
Code: d2 74 e4 0f 1f 40 00 eb ed 66 0f 1f 44 00 00 b8 01 00 00 00 5d c3 90 0f 1f 44 00 00 31 c0 ba 01 00 00 00 f0 0f b1 17 85 c0 75 01 <c3> 55 89 c6 48 89 e5 e8 04 25 ff ff 5d c3 0f 1f 40 00 66 2e 0f
Complete report #59265
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_lock
|
vmlinux |
0xffffffffa256a3f0
Build id: 3.10.0-957.el7.x86_64
|
0x10
|
2
|
|
vmlinux |
0xffffffffa205e26e
Build id: 3.10.0-957.el7.x86_64
|
0x1e
|
3
|
nfs_delegation_grab_inode
|
nfsv4 |
0xffffffffc088e7ba
Build id: 3.10.0-957.el7.x86_64
|
0x2a
|
4
|
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc088fb3c
Build id: 3.10.0-957.el7.x86_64
|
0xac
|
5
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc0889e48
Build id: 3.10.0-957.el7.x86_64
|
0x138
|
6
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc088bf80
Build id: 3.10.0-957.el7.x86_64
|
0x5d0
|
7
|
|
nfsv4 |
0xffffffffc088c280
Build id: 3.10.0-957.el7.x86_64
|
0x8d0
|
8
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc088c29f
Build id: 3.10.0-957.el7.x86_64
|
0x1f
|
9
|
kthread
|
vmlinux |
0xffffffffa1ec1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
10
|
|
vmlinux |
0xffffffffa1ec1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
11
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa2574c1d
Build id: 3.10.0-957.el7.x86_64
|
0x7
|
12
|
|
vmlinux |
0xffffffffa1ec1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 67s! [192.168.30.109-:1401]
Modules linked in: nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ext4 mbcache jbd2 iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi xt_recent nf_conntrack_netlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_security ebtable_filter ebtables ip6table_filter ip6_tables xt_set xt_addrtype xt_nat xt_statistic ipt_rpfilter iptable_raw xt_CHECKSUM ipt_REJECT nf_reject_ipv4 xt_mark xt_multiport xt_comment xt_conntrack iptable_filter iptable_mangle tcp_diag inet_diag ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack ip_tables binfmt_misc ip_set_hash_ip
ip_set_hash_net ip_set vxlan ip6_udp_tunnel udp_tunnel veth nfnetlink br_netfilter overlay(T) tun bridge stp llc devlink vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev sg pcspkr vmw_vmci parport_pc parport i2c_piix4 xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel drm_kms_helper serio_raw syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci vmxnet3 drm libahci ata_piix libata vmw_pvscsi drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod [last unloaded: iptable_mangle]
CPU: 5 PID: 1401 Comm: 192.168.30.109- Kdump: loaded Tainted: G L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9f6e4fa12080 ti: ffff9f71aebe0000 task.ti: ffff9f71aebe0000
RIP: 0010:[<ffffffffa256a3f0>] [<ffffffffa256a3f0>] _raw_spin_lock+0x10/0x30
RSP: 0018:ffff9f71aebe3d70 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff9f71a863ec00 RCX: ffff9f65585de000
RDX: 0000000000000001 RSI: 0000000000000051 RDI: ffff9f66cb3fa728
RBP: ffff9f71aebe3d88 R08: ffff9f65585da08c R09: 000000018020001f
R10: 0000000000000001 R11: fffff53dffa18f00 R12: ffff9f71a863ece0
R13: ffffffffa1ea9bdb R14: ffff9f71aebe3d00 R15: 000000018020001f
FS: 0000000000000000(0000) GS:ffff9f71ffd40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c0003b4018 CR3: 00000008bf410000 CR4: 00000000007607e0
PKRU: 00000000
Call Trace:
[<ffffffffa205e26e>] ? igrab+0x1e/0x60
[<ffffffffc088e7ba>] nfs_delegation_grab_inode+0x2a/0x50 [nfsv4]
[<ffffffffc088fb3c>] nfs_delegation_reap_unclaimed+0xac/0x140 [nfsv4]
[<ffffffffc0889e48>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc088bf80>] nfs4_state_manager+0x5d0/0x8d0 [nfsv4]
[<ffffffffc088c280>] ? nfs4_state_manager+0x8d0/0x8d0 [nfsv4]
[<ffffffffc088c29f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffffa1ec1c31>] kthread+0xd1/0xe0
[<ffffffffa1ec1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2574c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa1ec1b60>] ? insert_kthread_work+0x40/0x40
Code: 44 00 00 85 d2 74 e4 0f 1f 40 00 eb ed 66 0f 1f 44 00 00 b8 01 00 00 00 5d c3 90 0f 1f 44 00 00 31 c0 ba 01 00 00 00 f0 0f b1 17 <85> c0 75 01 c3 55 89 c6 48 89 e5 e8 40 1b ff ff 5d c3 0f 1f 40
Complete report #20214
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
_raw_spin_lock
|
vmlinux |
0xffffffffb431b730
Build id: 3.10.0-862.14.4.el7.x86_64
|
-
|
2
|
?
nfs_delegation_grab_inode
|
nfsv4 |
0xffffffffc1e6799b
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x1b
|
3
|
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc1e68d2c
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xac
|
4
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc1e63038
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x138
|
5
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc1e65170
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x5d0
|
6
|
|
nfsv4 |
0xffffffffc1e65470
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x8d0
|
7
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc1e6548f
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x1f
|
8
|
kthread
|
vmlinux |
0xffffffffb3cbdf21
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xd1
|
9
|
|
vmlinux |
0xffffffffb3cbde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
10
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb43255f7
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x21
|
11
|
|
vmlinux |
0xffffffffb3cbde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [129.112.5.24-ma:17524]
Modules linked in: nvidia_uvm(OE) tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) nvidia_modeset(POE) nvidia(POE) eeepc_wmi asus_wmi sparse_keymap iTCO_wdt rfkill iTCO_vendor_support video mxm_wmi
raid456 async_raid6_recov async_memcpy async_pq intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel raid6_pq snd_hda_codec_hdmi libcrc32c async_xor raid0 xor async_tx aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_hda_codec_realtek pcspkr snd_hda_codec_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops snd_hda_intel drm snd_hda_codec snd_seq snd_hda_core snd_seq_device snd_hwdep ipmi_devintf ipmi_msghandler snd_pcm sg mei_me snd_timer mei snd lpc_ich shpchp i2c_i801 soundcore wmi ip_tables ext4 mbcache jbd2 uas usb_storage sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel igb serio_raw ahci i2c_algo_bit dca libahci e1000e i2c_core libata ptp pps_core
CPU: 5 PID: 17524 Comm: 129.112.5.24-ma Kdump: loaded Tainted: P OE ------------ 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: ASUS All Series/X99-E WS/USB 3.1, BIOS 3201 06/17/2016
task: ffffa05dbeba6eb0 ti: ffffa03e116a8000 task.ti: ffffa03e116a8000
RIP: 0010:[<ffffffffb431b730>] [<ffffffffb431b730>] _raw_spin_lock+0x0/0x30
RSP: 0018:ffffa03e116abd90 EFLAGS: 00000202
RAX: 0000000000000001 RBX: ffffa05dd687a08c RCX: ffffa05dd3270000
RDX: 0000000000000009 RSI: 0000000000000009 RDI: ffffa05d777f8ed0
RBP: ffffa03e116abda8 R08: ffffa05dd687a08c R09: 0000000180200014
R10: 00000000dc712c01 R11: fffff7144071c400 R12: 00000000dc712c01
R13: fffff7144071c400 R14: ffffa05ddc713e00 R15: ffffa03e116abd88
FS: 0000000000000000(0000) GS:ffffa05ddf340000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f714a38f270 CR3: 0000001cf060e000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc1e6799b>] ? nfs_delegation_grab_inode+0x1b/0x50 [nfsv4]
[<ffffffffc1e68d2c>] nfs_delegation_reap_unclaimed+0xac/0x140 [nfsv4]
[<ffffffffc1e63038>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc1e65170>] nfs4_state_manager+0x5d0/0x8d0 [nfsv4]
[<ffffffffc1e65470>] ? nfs4_state_manager+0x8d0/0x8d0 [nfsv4]
[<ffffffffc1e6548f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffffb3cbdf21>] kthread+0xd1/0xe0
[<ffffffffb3cbde50>] ? insert_kthread_work+0x40/0x40
[<ffffffffb43255f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb3cbde50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 f0 0f b1 0f 39 c2 74 19 31 c0 5d c3 0f 1f 44 00 00 85 d2 74 e4 0f 1f 40 00 eb ed 66 0f 1f 44 00 00 b8 01 00 00 00 5d c3 90 <0f> 1f 44 00 00 31 c0 ba 01 00 00 00 f0 0f b1 17 85 c0 75 01 c3
Complete report #20047
Frame # |
Function |
Binary |
Source or offset |
Line |
1
|
igrab
|
vmlinux |
0xffffffffaba3bf73
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x53
|
2
|
?
nfs_delegation_grab_inode
|
nfsv4 |
0xffffffffc1fa39aa
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x2a
|
3
|
nfs_delegation_reap_unclaimed
|
nfsv4 |
0xffffffffc1fa4d2c
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xac
|
4
|
nfs4_state_clear_reclaim_reboot
|
nfsv4 |
0xffffffffc1f9f038
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x138
|
5
|
nfs4_state_manager
|
nfsv4 |
0xffffffffc1fa1170
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x5d0
|
6
|
|
nfsv4 |
0xffffffffc1fa1470
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x8d0
|
7
|
nfs4_run_state_manager
|
nfsv4 |
0xffffffffc1fa148f
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x1f
|
8
|
kthread
|
vmlinux |
0xffffffffab8bdf21
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xd1
|
9
|
|
vmlinux |
0xffffffffab8bde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
10
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffabf255f7
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x21
|
11
|
|
vmlinux |
0xffffffffab8bde50
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [129.112.5.24-ma:22902]
Modules linked in: tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) nvidia(POE) iTCO_wdt eeepc_wmi iTCO_vendor_support asus_wmi sparse_keymap rfkill video mxm_wmi
intel_powerclamp coretemp intel_rapl iosf_mbi raid456 kvm async_raid6_recov async_memcpy async_pq snd_hda_codec_hdmi irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd raid6_pq snd_hda_codec_realtek raid0 libcrc32c snd_hda_codec_generic async_xor xor async_tx pcspkr drm_kms_helper snd_hda_intel snd_hda_codec syscopyarea sysfillrect sysimgblt fb_sys_fops snd_hda_core drm snd_hwdep snd_seq i2c_i801 snd_seq_device ipmi_devintf ipmi_msghandler snd_pcm sg mei_me mei lpc_ich snd_timer snd shpchp soundcore wmi ip_tables ext4 mbcache jbd2 uas usb_storage sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw igb ahci e1000e libahci i2c_algo_bit dca i2c_core libata ptp pps_core
CPU: 11 PID: 22902 Comm: 129.112.5.24-ma Kdump: loaded Tainted: P OE ------------ 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: ASUS All Series/X99-E WS/USB 3.1, BIOS 3201 06/17/2016
task: ffff8a4b9a2e9fa0 ti: ffff8a3f86fa0000 task.ti: ffff8a3f86fa0000
RIP: 0010:[<ffffffffaba3bf73>] [<ffffffffaba3bf73>] igrab+0x53/0x60
RSP: 0018:ffff8a3f86fa3d90 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff8a4b51b0e08c RCX: ffff8a4b51b0e800
RDX: 0000000000000001 RSI: 0000000000000009 RDI: ffff8a4b9b0e0250
RBP: ffff8a3f86fa3da8 R08: ffff8a4b51b0e08c R09: 0000000100200010
R10: 000000004728fc01 R11: ffffed0bbf1ca300 R12: 000000004728fc01
R13: ffffed0bbf1ca300 R14: ffff8a4b4728fa00 R15: ffff8a3f86fa3d88
FS: 0000000000000000(0000) GS:ffff8a4b9f4c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005557c84a9300 CR3: 0000001f8b80e000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc1fa39aa>] ? nfs_delegation_grab_inode+0x2a/0x50 [nfsv4]
[<ffffffffc1fa4d2c>] nfs_delegation_reap_unclaimed+0xac/0x140 [nfsv4]
[<ffffffffc1f9f038>] nfs4_state_clear_reclaim_reboot+0x138/0x160 [nfsv4]
[<ffffffffc1fa1170>] nfs4_state_manager+0x5d0/0x8d0 [nfsv4]
[<ffffffffc1fa1470>] ? nfs4_state_manager+0x8d0/0x8d0 [nfsv4]
[<ffffffffc1fa148f>] nfs4_run_state_manager+0x1f/0x40 [nfsv4]
[<ffffffffab8bdf21>] kthread+0xd1/0xe0
[<ffffffffab8bde50>] ? insert_kthread_work+0x40/0x40
[<ffffffffabf255f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffab8bde50>] ? insert_kthread_work+0x40/0x40
Code: ff 83 30 01 00 00 4c 89 e7 c6 07 00 0f 1f 40 00 48 89 d8 5b 41 5c 5d c3 4c 89 e7 c6 07 00 0f 1f 40 00 31 db 48 89 d8 5b 41 5c 5d <c3> 66 90 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89