[ 215.820111] Running test [R:13235359 T:7 - RDMA Sanity Test - RoCE - Server - Kernel: 5.14.0-235.1827_748460310.el9.x86_64]
[-- MARK -- Mon Jan 16 15:40:00 2023]
[ 282.007694] systemd-rc-local-generator[29305]: /etc/rc.d/rc.local is not marked executable, skipping.
[ 286.433132] systemd-rc-local-generator[29372]: /etc/rc.d/rc.local is not marked executable, skipping.
[ 291.090918] systemd-rc-local-generator[29545]: /etc/rc.d/rc.local is not marked executable, skipping.
[ 430.662520] systemd-rc-local-generator[38027]: /etc/rc.d/rc.local is not marked executable, skipping.
[-- MARK -- Mon Jan 16 15:45:00 2023]
[ 655.609010] XFS (sda4): Unmounting Filesystem
[ 728.306027] XFS (sda4): Mounting V5 Filesystem
[ 728.445987] XFS (sda4): Ending clean mount
[ 728.956215] NFSD: Using nfsdcld client tracking operations.
[ 728.961832] NFSD: no clients to reclaim, skipping NFSv4 grace period (net f0000000)
[-- MARK -- Mon Jan 16 15:50:00 2023]
[ 987.144910] INFO: task nfsd:39296 blocked for more than 122 seconds.
[ 987.151316] Not tainted 5.14.0-235.1827_748460310.el9.x86_64 #1
[ 987.157805] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 987.165685] task:nfsd state:D stack: 0 pid:39296 ppid: 2 flags:0x00004000
[ 987.174096] Call Trace:
[ 987.176561]
[ 987.178678] __schedule+0x248/0x620
[ 987.182197] schedule+0x5a/0xc0
[ 987.185363] schedule_timeout+0x11d/0x160
[ 987.189403] __wait_for_common+0x93/0x1d0
[ 987.193441] ? usleep_range_state+0x90/0x90
[ 987.197658] svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma]
[ 987.203468] svc_rdma_recvfrom+0x217/0x470 [rpcrdma]
[ 987.208489] svc_handle_xprt+0x2ea/0x370 [sunrpc]
[ 987.213282] svc_recv+0xab/0x1b0 [sunrpc]
[ 987.217364] ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd]
[ 987.222487] nfsd+0xc3/0x190 [nfsd]
[ 987.226035] kthread+0xd9/0x100
[ 987.229202] ? kthread_complete_and_exit+0x20/0x20
[ 987.234027] ret_from_fork+0x22/0x30
[ 987.237633]
[ 987.239835] INFO: task nfsd:39297 blocked for more than 122 seconds.
[ 987.246232] Not tainted 5.14.0-235.1827_748460310.el9.x86_64 #1
[ 987.252717] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 987.260598] task:nfsd state:D stack: 0 pid:39297 ppid: 2 flags:0x00004000
[ 987.269007] Call Trace:
[ 987.271473]
[ 987.273590] __schedule+0x248/0x620
[ 987.277106] schedule+0x5a/0xc0
[ 987.280272] schedule_timeout+0x11d/0x160
[ 987.284311] __wait_for_common+0x93/0x1d0
[ 987.288350] ? usleep_range_state+0x90/0x90
[ 987.292565] svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma]
[ 987.298373] svc_rdma_recvfrom+0x217/0x470 [rpcrdma]
[ 987.303390] svc_handle_xprt+0x2ea/0x370 [sunrpc]
[ 987.308172] svc_recv+0xab/0x1b0 [sunrpc]
[ 987.312252] ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd]
[ 987.317374] nfsd+0xc3/0x190 [nfsd]
[ 987.320920] kthread+0xd9/0x100
[ 987.324084] ? kthread_complete_and_exit+0x20/0x20
[ 987.328913] ret_from_fork+0x22/0x30
[ 987.332518]
[ 987.334736] INFO: task nfsd:39298 blocked for more than 123 seconds.
[ 987.341142] Not tainted 5.14.0-235.1827_748460310.el9.x86_64 #1
[ 987.347627] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 987.355507] task:nfsd state:D stack: 0 pid:39298 ppid: 2 flags:0x00004000
[ 987.363921] Call Trace:
[ 987.366386]
[ 987.368503] __schedule+0x248/0x620
[ 987.372023] schedule+0x5a/0xc0
[ 987.375190] schedule_timeout+0x11d/0x160
[ 987.379232] __wait_for_common+0x93/0x1d0
[ 987.383273] ? usleep_range_state+0x90/0x90
[ 987.387491] svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma]
[ 987.393300] svc_rdma_recvfrom+0x217/0x470 [rpcrdma]
[ 987.398320] svc_handle_xprt+0x2ea/0x370 [sunrpc]
[ 987.403103] svc_recv+0xab/0x1b0 [sunrpc]
[ 987.407184] ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd]
[ 987.412305] nfsd+0xc3/0x190 [nfsd]
[ 987.415852] kthread+0xd9/0x100
[ 987.419018] ? kthread_complete_and_exit+0x20/0x20
[ 987.423846] ret_from_fork+0x22/0x30
[ 987.427453]
[ 987.429656] INFO: task nfsd:39300 blocked for more than 123 seconds.
[ 987.436052] Not tainted 5.14.0-235.1827_748460310.el9.x86_64 #1
[ 987.442537] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 987.450417] task:nfsd state:D stack: 0 pid:3[ 1110.026491] INFO: task nfsd:39296 blocked for more than 245 seconds.
[ 1110.032895] Not tainted 5.14.0-235.1827_748460310.el9.x86_64 #1
[ 1110.039380] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1110.047264] task:nfsd state:D stack: 0 pid:39296 ppid: 2 flags:0x00004000
[ 1110.055674] Call Trace:
[ 1110.058139]
[ 1110.060257] __schedule+0x248/0x620
[ 1110.063776] schedule+0x5a/0xc0
[ 1110.066943] schedule_timeout+0x11d/0x160
[ 1110.070983] __wait_for_common+0x93/0x1d0
[ 1110.075023] ? usleep_range_state+0x90/0x90
[ 1110.079243] svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma]
[ 1110.085053] svc_rdma_recvfrom+0x217/0x470 [rpcrdma]
[ 1110.090074] svc_handle_xprt+0x2ea/0x370 [sunrpc]
[ 1110.094869] svc_recv+0xab/0x1b0 [sunrpc]
[ 1110.098954] ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd]
[ 1110.104077] nfsd+0xc3/0x190 [nfsd]
[ 1110.107627] kthread+0xd9/0x100
[ 1110.110796] ? kthread_complete_and_exit+0x20/0x20
[ 1110.115623] ret_from_fork+0x22/0x30
[ 1110.119228]
[ 1110.121432] INFO: task nfsd:39297 blocked for more than 245 seconds.
[ 1110.127831] Not tainted 5.14.0-235.1827_748460310.el9.x86_64 #1
[ 1110.134317] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1110.142198] task:nfsd state:D stack: 0 pid:39297 ppid: 2 flags:0x00004000
[ 1110.150608] Call Trace:
[ 1110.153072]
[ 1110.155189] __schedule+0x248/0x620
[ 1110.158706] schedule+0x5a/0xc0
[ 1110.161873] schedule_timeout+0x11d/0x160
[ 1110.165914] __wait_for_common+0x93/0x1d0
[ 1110.169954] ? usleep_range_state+0x90/0x90
[ 1110.174170] svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma]
[ 1110.179978] svc_rdma_recvfrom+0x217/0x470 [rpcrdma]
[ 1110.184995] svc_handle_xprt+0x2ea/0x370 [sunrpc]
[ 1110.189775] svc_recv+0xab/0x1b0 [sunrpc]
[ 1110.193854] ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd]
[ 1110.198976] nfsd+0xc3/0x190 [nfsd]
[ 1110.202525] kthread+0xd9/0x100
[ 1110.205689] ? kthread_complete_and_exit+0x20/0x20
[ 1110.210516] ret_from_fork+0x22/0x30
[ 1110.214121]
[ 1110.216325] INFO: task nfsd:39298 blocked for more than 245 seconds.
[ 1110.222725] Not tainted 5.14.0-235.1827_748460310.el9.x86_64 #1
[ 1110.229212] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1110.237094] task:nfsd state:D stack: 0 pid:39298 ppid: 2 flags:0x00004000
[ 1110.245500] Call Trace:
[ 1110.247965]
[ 1110.250080] __schedule+0x248/0x620
[ 1110.253596] schedule+0x5a/0xc0
[ 1110.256764] schedule_timeout+0x11d/0x160
[ 1110.260807] __wait_for_common+0x93/0x1d0
[ 1110.264848] ? usleep_range_state+0x90/0x90
[ 1110.269066] svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma]
[ 1110.274872] svc_rdma_recvfrom+0x217/0x470 [rpcrdma]
[ 1110.279892] svc_handle_xprt+0x2ea/0x370 [sunrpc]
[ 1110.284674] svc_recv+0xab/0x1b0 [sunrpc]
[ 1110.288753] ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd]
[ 1110.293872] nfsd+0xc3/0x190 [nfsd]
[ 1110.297420] kthread+0xd9/0x100
[ 1110.300589] ? kthread_complete_and_exit+0x20/0x20
[ 1110.305418] ret_from_fork+0x22/0x30
[ 1110.309025]
[ 1110.311230] INFO: task nfsd:39300 blocked for more than 246 seconds.
[ 1110.317627] Not tainted 5.14.0-235.1827_748460310.el9.x86_64 #1
[ 1110.324110] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1110.331990] task:nfsd state:D stack: 0 pid:39300 ppid: 2 flags:0x00004000
[ 1110.340400] Call Trace:
[ 1110.342870]
[ 1110.344988] __schedule+0x248/0x620
[ 1110.348508] schedule+0x5a/0xc0
[ 1110.351674] schedule_timeout+0x11d/0x160
[ 1110.355713] __wait_for_common+0x93/0x1d0
[ 1110.359753] ? usleep_range_state+0x90/0x90
[ 1110.363970] svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma]
[ 1110.369775] svc_rdma_recvfrom+0x217/0x470 [rpcrdma]
[ 1110.374794] svc_handle_xprt+0x2ea/0x370 [sunrpc]
[ 1110.379575] svc_recv+0xab/0x1b0 [sunrpc]
[ 1110.383656] ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd]
[ 1110.388777] nfsd+0xc3/0x190 [nfsd]
[ 1110.392326] kthread+0xd9/0x100
[ 1110.395491] ? kthread_complete_and_exit+0x20/0x20
[ 1110.400316] ret_from_fork+0x22/0x30
[ 1110.403921]
[ 1110.406127] INFO: task nfsd:39302 blocked for more than 246 seconds.
[ 1110.412526] Not tainted 5.14.0-235.1827_748460310.el9.x86_64 #1
[ 1110.419014] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1110.426896] task:nfsd state:D stack: 0 pid:39302 ppid: 2 flags:0x00004000
[ 1110.435304] Call Trace:
[ 1110.437771]
[ 1110.439890] __schedule+0x248/0x620
[ 1110.443410] schedule+0x5a/0xc0
[ 1110.446578] schedule_timeout+0x11d/0x160
[ 1110.450618] __wait_for_common+0x93/0x1d0
[ 1110.454660] ? usleep_range_state+0x90/0x90
[ 1110.458880] svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma]
[ 1110.464686] svc_rdma_recvfrom+0x217/0x470 [rpcrdma]
[ 1110.469710] svc_handle_xprt+0x2ea/0x370 [sunrpc]
[ 1110.474489] svc_recv+0xab/0x1b0 [sunrpc]
[ 1110.478571] ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd]
[ 1110.483693] nfsd+0xc3/0x190 [nfsd]
[ 1110.487240] kthread+0xd9/0x100
[ 1110.490406] ? kthread_complete_and_exit+0x20/0x20
[ 1110.495233] ret_from_fork+0x22/0x30
[ 1110.498838]
[-- MARK -- Mon Jan 16 15:55:00 2023]
[-- MARK -- Mon Jan 16 16:00:00 2023]
[-- MARK -- Mon Jan 16 16:05:00 2023]
[-- MARK -- Mon Jan 16 16:10:00 2023]
[-- MARK -- Mon Jan 16 16:15:00 2023]
[-- MARK -- Mon Jan 16 16:20:00 2023]
[-- MARK -- Mon Jan 16 16:25:00 2023]
[-- MARK -- Mon Jan 16 16:30:00 2023]
[-- MARK -- Mon Jan 16 16:35:00 2023]