[  224.144344] Running test [R:13153658 T:7 - RDMA Sanity Test - RoCE - Server - Kernel: 5.14.0-226.rt14.227.1773_732124971.el9.x86_64] 
[-- MARK -- Mon Dec 26 15:55:00 2022] 
[  367.292394] systemd-rc-local-generator[29869]: /etc/rc.d/rc.local is not marked executable, skipping. 
[  371.974922] systemd-rc-local-generator[29941]: /etc/rc.d/rc.local is not marked executable, skipping. 
[  376.627355] systemd-rc-local-generator[30116]: /etc/rc.d/rc.local is not marked executable, skipping. 
[  454.455014] systemd-rc-local-generator[38575]: /etc/rc.d/rc.local is not marked executable, skipping. 
[-- MARK -- Mon Dec 26 16:00:00 2022] 
[  703.205100] XFS (sda4): Unmounting Filesystem 
[  776.004664] XFS (sda4): Mounting V5 Filesystem 
[  776.139906] XFS (sda4): Ending clean mount 
[  776.694036] NFSD: Using nfsdcld client tracking operations. 
[  776.694041] NFSD: no clients to reclaim, skipping NFSv4 grace period (net f0000000) 
[-- MARK -- Mon Dec 26 16:05:00 2022] 
[  987.059992] INFO: task nfsd:39854 blocked for more than 122 seconds. 
[  987.066397]       Not tainted 5.14.0-226.rt14.227.1773_732124971.el9.x86_64 #1 
[  987.073662] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[  987.081540] task:nfsd            state:D stack:    0 pid:39854 ppid:     2 flags:0x00004000 
[  987.089945] Call Trace: 
[  987.092406]  <TASK> 
[  987.094521]  __schedule+0x25b/0x660 
[  987.098040]  schedule+0x5d/0xd0 
[  987.101209]  schedule_timeout+0x151/0x190 
[  987.105246]  ? preempt_count_add+0x70/0xa0 
[  987.109371]  __wait_for_common+0x8e/0x1c0 
[  987.113408]  ? usleep_range_state+0x90/0x90 
[  987.117623]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[  987.123425]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[  987.128437]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[  987.133217]  svc_recv+0xa1/0x1d0 [sunrpc] 
[  987.137287]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[  987.142396]  nfsd+0xc3/0x190 [nfsd] 
[  987.145931]  kthread+0x106/0x130 
[  987.149184]  ? kthread_complete_and_exit+0x20/0x20 
[  987.154005]  ret_from_fork+0x22/0x30 
[  987.157610]  </TASK> 
[  987.159810] INFO: task nfsd:39855 blocked for more than 122 seconds. 
[  987.166210]       Not tainted 5.14.0-226.rt14.227.1773_732124971.el9.x86_64 #1 
[  987.173477] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[  987.181353] task:nfsd            state:D stack:    0 pid:39855 ppid:     2 flags:0x00004000 
[  987.189757] Call Trace: 
[  987.192217]  <TASK> 
[  987.194329]  __schedule+0x25b/0x660 
[  987.197840]  schedule+0x5d/0xd0 
[  987.201003]  schedule_timeout+0x151/0x190 
[  987.205038]  ? preempt_count_add+0x70/0xa0 
[  987.209164]  __wait_for_common+0x8e/0x1c0 
[  987.213197]  ? usleep_range_state+0x90/0x90 
[  987.217407]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[  987.223205]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[  987.228215]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[  987.232983]  svc_recv+0xa1/0x1d0 [sunrpc] 
[  987.237047]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[  987.242154]  nfsd+0xc3/0x190 [nfsd] 
[  987.245689]  kthread+0x106/0x130 
[  987.248937]  ? kthread_complete_and_exit+0x20/0x20 
[  987.253761]  ret_from_fork+0x22/0x30 
[  987.257375]  </TASK> 
[  987.259578] INFO: task nfsd:39857 blocked for more than 123 seconds. 
[  987.265972]       Not tainted 5.14.0-226.rt14.227.1773_732124971.el9.x86_64 #1 
[  987.273237] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[  987.281116] task:nfsd            state:D stack:    0 pid:39857 ppid:     2 flags:0x00004000 
[  987.289521] Call Trace: 
[  987.291990]  <TASK> 
[  987.294102]  __schedule+0x25b/0x660 
[  987.297613]  schedule+0x5d/0xd0 
[  987.300778]  schedule_timeout+0x151/0x190 
[  987.304815]  ? preempt_count_add+0x70/0xa0 
[  987.308938]  __wait_for_common+0x8e/0x1c0 
[  987.312974]  ? usleep_range_state+0x90/0x90 
[  987.317184]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[  987.322982]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[  987.327993]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[  987.332763]  svc_recv+0xa1/0x1d0 [sunrpc] 
[  987.336841]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[  987.341947]  nfsd+0xc3/0x190 [nfsd] 
[  987.345482]  kthread+0x106/0x130 
[  987.348732]  ? kthread_complete_and_exit+0x20/0x20 
[  987.353555]  ret_from_fork+0x22/0x30 
[  987.357158]  </TASK> 
[  987.359358] INFO: task nfsd:39860 blocked for more than 123 seconds. 
[  987.365751]       Not tainted 5.14.0-226.rt14.227.1773_732124971.el9.x86_64 #1 
[  987.373017] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[  987.380896] task:nfsd            state:D stack:    0 pid:39860 ppid:     2 flags:0x00004000 
[  987.389298] Call Trace: 
[  987.391761]  <TASK> 
[  987.393872]  __schedule+0x25b/0x660 
[  987.397386]  schedule+0x5d/0xd0 
[  987.400548]  schedule_timeout+0x151/0x190 
[  987.404586]  ? preempt_count_add+0x70/0xa0 
[  987.408707]  __wait_for_common+0x8e/0x1c0 
[  987.412741]  ? usleep_range_state+0x90/0x90 
[  987.416954]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[  987.422757]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[  987.427786]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[  987.432553]  svc_recv+0xa1/0x1d0 [sunrpc] 
[  987.436620]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[  987.441725]  nfsd+0xc3/0x190 [nfsd] 
[  987.445260]  kthread+0x106/0x130 
[  987.448508]  ? kthread_complete_and_exit+0x20/0x20 
[  987.453329]  ret_from_fork+0x22/0x30 
[  987.456928]  </TASK> 
[  987.459131] INFO: task rpc.nfsd:40001 blocked for more than 123 seconds. 
[  987.465872]       Not tainted 5.14.0-226.rt14.227.1773_732124971.el9.x86_64 #1 
[  987.473140] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[  987.481015] task:rpc.nfsd        state:D stack:    0 pid:40001 ppid:     1 flags:0x00000006 
[  987.489424] Call Trace: 
[  987.491885]  <TASK> 
[  987.493997]  __schedule+0x25b/0x660 
[  987.497510]  schedule+0x5d/0xd0 
[  987.500670]  schedule_timeout+0x151/0x190 
[  987.504708]  ? default_send_IPI_single_phys+0x26/0x40 
[  987.509792]  ? preempt_count_add+0x70/0xa0 
[  987.513914]  __wait_for_common+0x8e/0x1c0 
[  987.517949]  ? usleep_range_state+0x90/0x90 
[  987.522162]  kthread_stop+0x62/0x1d0 
[  987.525764]  svc_set_num_threads+0x10d/0x200 [sunrpc] 
[  987.530893]  nfsd_svc+0x123/0x190 [nfsd] 
[  987.534867]  write_threads+0x95/0x100 [nfsd] 
[  987.539188]  ? _copy_from_user+0x3a/0x60 
[  987.543138]  ? simple_transaction_get+0xc4/0xf0 
[  987.547700]  ? write_pool_threads+0x230/0x230 [nfsd] 
[  987.552723]  nfsctl_transaction_write+0x46/0x80 [nfsd] 
[  987.557915]  vfs_write+0xb5/0x2e0 
[  987.561254]  ksys_write+0x62/0xe0 
[  987.564591]  do_syscall_64+0x5c/0x90 
[  987.568190]  ? __rseq_handle_notify_resume+0x32/0x50 
[  987.573186]  ? exit_to_user_mode_loop+0xd7/0x130 
[  987.577835]  ? exit_to_user_mode_prepare+0xa8/0xf0 
[  987.582657]  ? syscall_exit_to_user_mode+0x12/0x30 
[  987.587477]  ? do_syscall_64+0x69/0x90 
[  987.591250]  ? do_user_addr_fault+0x1e5/0x690 
[  987.595638]  ? exc_page_fault+0x65/0x180 
[  987.599588]  entry_SYSCALL_64_after_hwframe+0x63/0xcd 
[  987.604671] RIP: 0033:0x7f30ab485be7 
[  987.608269] RSP: 002b:00007ffc5d54ec38 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 
[  987.615893] RAX: ffffffffffffffda RBX: 0000000000000004 RCX: 00007f30ab485be7 
[  987.623073] RDX: 0000000000000002 RSI: 0000563fdc249c20 RDI: 0000000000000003 
[  987.630251] RBP: 0000000000000003 R08: 0000000000000000 R09: 00007ffc5d54ead0 
[  987.637431] R10: 0000000000000000 R11: 0000000000000246 R12: 0000563fdc249c20 
[  987.644611] R13: 00007f30ab3446c0 R14: 00007ffc5d54ed00 R15: 00000000ffffffff 
[  987.651793]  </TASK> 
[ 1109.941681] INFO: task nfsd:39854 blocked for more than 245 seconds. 
[ 1109.948086]       Not tainted 5.14.0-226.rt14.227.1773_732124971.el9.x86_64 #1 
[ 1109.955351] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1109.963227] task:nfsd            state:D stack:    0 pid:39854 ppid:     2 flags:0x00004000 
[ 1109.971633] Call Trace: 
[ 1109.974094]  <TASK> 
[ 1109.976215]  __schedule+0x25b/0x660 
[ 1109.979732]  schedule+0x5d/0xd0 
[ 1109.982893]  schedule_timeout+0x151/0x190 
[ 1109.986931]  ? preempt_count_add+0x70/0xa0 
[ 1109.991056]  __wait_for_common+0x8e/0x1c0 
[ 1109.995093]  ? usleep_range_state+0x90/0x90 
[ 1109.999309]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1110.005111]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1110.010124]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1110.014901]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1110.018973]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1110.024085]  nfsd+0xc3/0x190 [nfsd] 
[ 1110.027623]  kthread+0x106/0x130 
[ 1110.030872]  ? kthread_complete_and_exit+0x20/0x20 
[ 1110.035693]  ret_from_fork+0x22/0x30 
[ 1110.039302]  </TASK> 
[ 1110.041501] INFO: task nfsd:39855 blocked for more than 245 seconds. 
[ 1110.047894]       Not tainted 5.14.0-226.rt14.227.1773_732124971.el9.x86_64 #1 
[ 1110.055158] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1110.063036] task:nfsd            state:D stack:    0 pid:39855 ppid:     2 flags:0x00004000 
[ 1110.071442] Call Trace: 
[ 1110.073902]  <TASK> 
[ 1110.076015]  __schedule+0x25b/0x660 
[ 1110.079525]  schedule+0x5d/0xd0 
[ 1110.082687]  schedule_timeout+0x151/0x190 
[ 1110.086721]  ? preempt_count_add+0x70/0xa0 
[ 1110.090843]  __wait_for_common+0x8e/0x1c0 
[ 1110.094878]  ? usleep_range_state+0x90/0x90 
[ 1110.099087]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1110.104890]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1110.109901]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1110.114671]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1110.118739]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1110.123845]  nfsd+0xc3/0x190 [nfsd] 
[ 1110.127378]  kthread+0x106/0x130 
[ 1110.130628]  ? kthread_complete_and_exit+0x20/0x20 
[ 1110.135449]  ret_from_fork+0x22/0x30 
[ 1110.139062]  </TASK> 
[ 1110.141265] INFO: task nfsd:39857 blocked for more than 245 seconds. 
[ 1110.147661]       Not tainted 5.14.0-226.rt14.227.1773_732124971.el9.x86_64 #1 
[ 1110.154928] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1110.162806] task:nfsd            state:D stack:    0 pid:39857 ppid:     2 flags:0x00004000 
[ 1110.171214] Call Trace: 
[ 1110.173676]  <TASK> 
[ 1110.175789]  __schedule+0x25b/0x660 
[ 1110.179304]  schedule+0x5d/0xd0 
[ 1110.182465]  schedule_timeout+0x151/0x190 
[ 1110.186501]  ? preempt_count_add+0x70/0xa0 
[ 1110.190625]  __wait_for_common+0x8e/0x1c0 
[ 1110.194659]  ? usleep_range_state+0x90/0x90 
[ 1110.198869]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1110.204665]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1110.209674]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1110.214442]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1110.218528]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1110.223635]  nfsd+0xc3/0x190 [nfsd] 
[ 1110.227169]  kthread+0x106/0x130 
[ 1110.230417]  ? kthread_complete_and_exit+0x20/0x20 
[ 1110.235244]  ret_from_fork+0x22/0x30 
[ 1110.238847]  </TASK> 
[ 1110.241047] INFO: task nfsd:39860 blocked for more than 246 seconds. 
[ 1110.247439]       Not tainted 5.14.0-226.rt14.227.1773_732124971.el9.x86_64 #1 
[ 1110.254704] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1110.262582] task:nfsd            state:D stack:    0 pid:39860 ppid:     2 flags:0x00004000 
[ 1110.270986] Call Trace: 
[ 1110.273448]  <TASK> 
[ 1110.275573]  __schedule+0x25b/0x660 
[ 1110.279085]  schedule+0x5d/0xd0 
[ 1110.282246]  schedule_timeout+0x151/0x190 
[ 1110.286283]  ? preempt_count_add+0x70/0xa0 
[ 1110.290406]  __wait_for_common+0x8e/0x1c0 
[ 1110.294442]  ? usleep_range_state+0x90/0x90 
[ 1110.298657]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1110.304452]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1110.309470]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1110.314239]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1110.318307]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1110.323414]  nfsd+0xc3/0x190 [nfsd] 
[ 1110.326954]  kthread+0x106/0x130 
[ 1110.330204]  ? kthread_complete_and_exit+0x20/0x20 
[ 1110.335025]  ret_from_fork+0x22/0x30 
[ 1110.338626]  </TASK> 
[ 1110.340828] INFO: task rpc.nfsd:40001 blocked for more than 246 seconds. 
[ 1110.347571]       Not tainted 5.14.0-226.rt14.227.1773_732124971.el9.x86_64 #1 
[ 1110.354839] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1110.362722] task:rpc.nfsd        state:D stack:    0 pid:40001 ppid:     1 flags:0x00000006 
[ 1110.371127] Call Trace: 
[ 1110.373588]  <TASK> 
[ 1110.375700]  __schedule+0x25b/0x660 
[ 1110.379214]  schedule+0x5d/0xd0 
[ 1110.382375]  schedule_timeout+0x151/0x190 
[ 1110.386411]  ? default_send_IPI_single_phys+0x26/0x40 
[ 1110.391496]  ? preempt_count_add+0x70/0xa0 
[ 1110.395618]  __wait_for_common+0x8e/0x1c0 
[ 1110.399655]  ? usleep_range_state+0x90/0x90 
[ 1110.403865]  kthread_stop+0x62/0x1d0 
[ 1110.407467]  svc_set_num_threads+0x10d/0x200 [sunrpc] 
[ 1110.412587]  nfsd_svc+0x123/0x190 [nfsd] 
[ 1110.416561]  write_threads+0x95/0x100 [nfsd] 
[ 1110.420883]  ? _copy_from_user+0x3a/0x60 
[ 1110.424831]  ? simple_transaction_get+0xc4/0xf0 
[ 1110.429400]  ? write_pool_threads+0x230/0x230 [nfsd] 
[ 1110.434417]  nfsctl_transaction_write+0x46/0x80 [nfsd] 
[ 1110.439612]  vfs_write+0xb5/0x2e0 
[ 1110.442951]  ksys_write+0x62/0xe0 
[ 1110.446287]  do_syscall_64+0x5c/0x90 
[ 1110.449887]  ? __rseq_handle_notify_resume+0x32/0x50 
[ 1110.454883]  ? exit_to_user_mode_loop+0xd7/0x130 
[ 1110.459530]  ? exit_to_user_mode_prepare+0xa8/0xf0 
[ 1110.464352]  ? syscall_exit_to_user_mode+0x12/0x30 
[ 1110.469173]  ? do_syscall_64+0x69/0x90 
[ 1110.472946]  ? do_user_addr_fault+0x1e5/0x690 
[ 1110.477331]  ? exc_page_fault+0x65/0x180 
[ 1110.481281]  entry_SYSCALL_64_after_hwframe+0x63/0xcd 
[ 1110.486367] RIP: 0033:0x7f30ab485be7 
[ 1110.489965] RSP: 002b:00007ffc5d54ec38 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 
[ 1110.497588] RAX: ffffffffffffffda RBX: 0000000000000004 RCX: 00007f30ab485be7 
[ 1110.504768] RDX: 0000000000000002 RSI: 0000563fdc249c20 RDI: 0000000000000003 
[ 1110.511948] RBP: 0000000000000003 R08: 0000000000000000 R09: 00007ffc5d54ead0 
[ 1110.519132] R10: 0000000000000000 R11: 0000000000000246 R12: 0000563fdc249c20 
[ 1110.526310] R13: 00007f30ab3446c0 R14: 00007ffc5d54ed00 R15: 00000000ffffffff 
[ 1110.533493]  </TASK> 
[-- MARK -- Mon Dec 26 16:10:00 2022] 
[-- MARK -- Mon Dec 26 16:15:00 2022] 
[-- MARK -- Mon Dec 26 16:20:00 2022] 
[-- MARK -- Mon Dec 26 16:25:00 2022] 
[-- MARK -- Mon Dec 26 16:30:00 2022] 
[-- MARK -- Mon Dec 26 16:35:00 2022] 
[-- MARK -- Mon Dec 26 16:40:00 2022] 
[-- MARK -- Mon Dec 26 16:45:00 2022] 
[-- MARK -- Mon Dec 26 16:50:00 2022]