[  215.017669] Running test [R:13197902 T:7 - RDMA Sanity Test - RoCE - Server - Kernel: 5.14.0-231.1852_742217007.el9.x86_64] 
[  240.125358] systemd-rc-local-generator[14936]: /etc/rc.d/rc.local is not marked executable, skipping. 
[  241.612574] systemd-rc-local-generator[15085]: /etc/rc.d/rc.local is not marked executable, skipping. 
[-- MARK -- Mon Jan  9 21:45:00 2023] 
[  619.514956] systemd-rc-local-generator[30919]: /etc/rc.d/rc.local is not marked executable, skipping. 
[  623.759067] systemd-rc-local-generator[30989]: /etc/rc.d/rc.local is not marked executable, skipping. 
[  628.049979] systemd-rc-local-generator[31166]: /etc/rc.d/rc.local is not marked executable, skipping. 
[  699.445957] systemd-rc-local-generator[39733]: /etc/rc.d/rc.local is not marked executable, skipping. 
[-- MARK -- Mon Jan  9 21:50:00 2023] 
[  884.072850] XFS (sda4): Unmounting Filesystem 
[  956.796178] XFS (sda4): Mounting V5 Filesystem 
[  956.940426] XFS (sda4): Ending clean mount 
[  957.461913] NFSD: Using nfsdcld client tracking operations. 
[  957.467531] NFSD: no clients to reclaim, skipping NFSv4 grace period (net f0000000) 
[-- MARK -- Mon Jan  9 21:55:00 2023] 
[ 1233.171941] INFO: task nfsd:41446 blocked for more than 122 seconds. 
[ 1233.178350]       Not tainted 5.14.0-231.1852_742217007.el9.x86_64 #1 
[ 1233.184839] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1233.192724] task:nfsd            state:D stack:    0 pid:41446 ppid:     2 flags:0x00004000 
[ 1233.201165] Call Trace: 
[ 1233.203636]  <TASK> 
[ 1233.205760]  __schedule+0x248/0x620 
[ 1233.209290]  schedule+0x5a/0xc0 
[ 1233.212462]  schedule_timeout+0x11d/0x160 
[ 1233.216514]  __wait_for_common+0x93/0x1d0 
[ 1233.220557]  ? usleep_range_state+0x90/0x90 
[ 1233.224779]  svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma] 
[ 1233.230592]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1233.235628]  svc_handle_xprt+0x2ea/0x370 [sunrpc] 
[ 1233.240424]  svc_recv+0xab/0x1b0 [sunrpc] 
[ 1233.244507]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1233.249641]  nfsd+0xc3/0x190 [nfsd] 
[ 1233.253191]  kthread+0xd9/0x100 
[ 1233.256361]  ? kthread_complete_and_exit+0x20/0x20 
[ 1233.261187]  ret_from_fork+0x22/0x30 
[ 1233.264794]  </TASK> 
[ 1233.267011] INFO: task nfsd:41449 blocked for more than 122 seconds. 
[ 1233.273411]       Not tainted 5.14.0-231.1852_742217007.el9.x86_64 #1 
[ 1233.279897] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1233.287778] task:nfsd            state:D stack:    0 pid:41449 ppid:     2 flags:0x00004000 
[ 1233.296190] Call Trace: 
[ 1233.298673]  <TASK> 
[ 1233.300793]  __schedule+0x248/0x620 
[ 1233.304314]  schedule+0x5a/0xc0 
[ 1233.307484]  schedule_timeout+0x11d/0x160 
[ 1233.311526]  __wait_for_common+0x93/0x1d0 
[ 1233.315576]  ? usleep_range_state+0x90/0x90 
[ 1233.319795]  svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma] 
[ 1233.325607]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1233.330626]  svc_handle_xprt+0x2ea/0x370 [sunrpc] 
[ 1233.335417]  svc_recv+0xab/0x1b0 [sunrpc] 
[ 1233.339499]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1233.344621]  nfsd+0xc3/0x190 [nfsd] 
[ 1233.348178]  kthread+0xd9/0x100 
[ 1233.351347]  ? kthread_complete_and_exit+0x20/0x20 
[ 1233.356177]  ret_from_fork+0x22/0x30 
[ 1233.359786]  </TASK> 
[ 1233.362010] INFO: task rpc.nfsd:41699 blocked for more than 123 seconds. 
[ 1233.368764]       Not tainted 5.14.0-231.1852_742217007.el9.x86_64 #1 
[ 1233.375261] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1233.383141] task:rpc.nfsd        state:D stack:    0 pid:41699 ppid:     1 flags:0x00000006 
[ 1233.391562] Call Trace: 
[ 1233.394028]  <TASK> 
[ 1233.396143]  __schedule+0x248/0x620 
[ 1233.399663]  schedule+0x5a/0xc0 
[ 1233.402831]  schedule_timeout+0x11d/0x160 
[ 1233.406871]  ? default_send_IPI_single_phys+0x26/0x40 
[ 1233.411971]  ? ttwu_queue_wakelist+0xf4/0x110 
[ 1233.416363]  __wait_for_common+0x93/0x1d0 
[ 1233.420402]  ? usleep_range_state+0x90/0x90 
[ 1233.424615]  kthread_stop+0x62/0x170 
[ 1233.428228]  svc_set_num_threads+0x10d/0x1a0 [sunrpc] 
[ 1233.433362]  nfsd_svc+0x123/0x190 [nfsd] 
[ 1233.437346]  write_threads+0x95/0x100 [nfsd] 
[ 1233.441687]  ? _copy_from_user+0x3a/0x60 
[ 1233.445641]  ? simple_transaction_get+0xc4/0xf0 
[ 1233.450207]  ? write_pool_threads+0x230/0x230 [nfsd] 
[ 1233.455239]  nfsctl_transaction_write+0x46/0x80 [nfsd] 
[ 1233.460451]  vfs_write+0xb5/0x280 
[ 1233.463794]  ksys_write+0x5f/0xe0 
[ 1233.467136]  do_syscall_64+0x5c/0x90 
[ 1233.470740]  ? syscall_exit_work+0x11a/0x150 
[ 1233.475053]  ? syscall_exit_to_user_mode+0x12/0x30 
[ 1233.479877]  ? do_syscall_64+0x69/0x90 
[ 1233.483656]  ? do_syscall_64+0x69/0x90 
[ 1233.487434]  ? syscall_exit_work+0x11a/0x150 
[ 1233.491745]  ? syscall_exit_to_user_mode+0x12/0x30 
[ 1233.496571]  ? do_syscall_64+0x69/0x90 
[ 1233.500349]  ? exc_page_fault+0x62/0x150 
[ 1233.504303]  entry_SYSCALL_64_after_hwframe+0x63/0xcd 
[ 1233.509400] RIP: 0033:0x7f296033ebe7 
[ 1233.513004] RSP: 002b:00007ffd338ef548 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 
[ 1233.520625] RAX: ffffffffffffffda RBX: 0000000000000004 RCX: 00007f296033ebe7 
[ 1233.527821] RDX: 0000000000000002 RSI: 000055d77333ac20 RDI: 0000000000000003 
[ 1233.535002] RBP: 0000000000000003 R08: 0000000000000000 R09: 00007ffd338ef3e0 
[ 1233.542185] R10: 0000000000000000 R11: 0000000000000246 R12: 000055d77333ac20 
[ 1233.549367] R13: 00007f29604226c0 R14: 00007ffd338ef610 R15: 00000000ffffffff 
[ 1233.556566]  </TASK> 
[ 1356.053716] INFO: task nfsd:41446 blocked for more than 245 seconds. 
[ 1356.060126]       Not tainted 5.14.0-231.1852_742217007.el9.x86_64 #1 
[ 1356.066611] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1356.074495] task:nfsd            state:D stack:    0 pid:41446 ppid:     2 flags:0x00004000 
[ 1356.082908] Call Trace: 
[ 1356.085374]  <TASK> 
[ 1356.087493]  __schedule+0x248/0x620 
[ 1356.091018]  schedule+0x5a/0xc0 
[ 1356.094189]  schedule_timeout+0x11d/0x160 
[ 1356.098230]  __wait_for_common+0x93/0x1d0 
[ 1356.102271]  ? usleep_range_state+0x90/0x90 
[ 1356.106489]  svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma] 
[ 1356.112300]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1356.117324]  svc_handle_xprt+0x2ea/0x370 [sunrpc] 
[ 1356.122117]  svc_recv+0xab/0x1b0 [sunrpc] 
[ 1356.126200]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1356.131323]  nfsd+0xc3/0x190 [nfsd] 
[ 1356.134873]  kthread+0xd9/0x100 
[ 1356.138044]  ? kthread_complete_and_exit+0x20/0x20 
[ 1356.142869]  ret_from_fork+0x22/0x30 
[ 1356.146479]  </TASK> 
[ 1356.148684] INFO: task nfsd:41449 blocked for more than 245 seconds. 
[ 1356.155084]       Not tainted 5.14.0-231.1852_742217007.el9.x86_64 #1 
[ 1356.161570] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1356.169453] task:nfsd            state:D stack:    0 pid:41449 ppid:     2 flags:0x00004000 
[ 1356.177861] Call Trace: 
[ 1356.180327]  <TASK> 
[ 1356.182442]  __schedule+0x248/0x620 
[ 1356.185962]  schedule+0x5a/0xc0 
[ 1356.189130]  schedule_timeout+0x11d/0x160 
[ 1356.193170]  __wait_for_common+0x93/0x1d0 
[ 1356.197211]  ? usleep_range_state+0x90/0x90 
[ 1356.201426]  svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma] 
[ 1356.207234]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1356.212252]  svc_handle_xprt+0x2ea/0x370 [sunrpc] 
[ 1356.217034]  svc_recv+0xab/0x1b0 [sunrpc] 
[ 1356.221111]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1356.226232]  nfsd+0xc3/0x190 [nfsd] 
[ 1356.229779]  kthread+0xd9/0x100 
[ 1356.232944]  ? kthread_complete_and_exit+0x20/0x20 
[ 1356.237773]  ret_from_fork+0x22/0x30 
[ 1356.241378]  </TASK> 
[ 1356.243585] INFO: task rpc.nfsd:41699 blocked for more than 245 seconds. 
[ 1356.250338]       Not tainted 5.14.0-231.1852_742217007.el9.x86_64 #1 
[ 1356.256822] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1356.264705] task:rpc.nfsd        state:D stack:    0 pid:41699 ppid:     1 flags:0x00000006 
[ 1356.273114] Call Trace: 
[ 1356.275579]  <TASK> 
[ 1356.277698]  __schedule+0x248/0x620 
[ 1356.281216]  schedule+0x5a/0xc0 
[ 1356.284383]  schedule_timeout+0x11d/0x160 
[ 1356.288424]  ? default_send_IPI_single_phys+0x26/0x40 
[ 1356.293516]  ? ttwu_queue_wakelist+0xf4/0x110 
[ 1356.297907]  __wait_for_common+0x93/0x1d0 
[ 1356.301946]  ? usleep_range_state+0x90/0x90 
[ 1356.306162]  kthread_stop+0x62/0x170 
[ 1356.309765]  svc_set_num_threads+0x10d/0x1a0 [sunrpc] 
[ 1356.314896]  nfsd_svc+0x123/0x190 [nfsd] 
[ 1356.318879]  write_threads+0x95/0x100 [nfsd] 
[ 1356.323210]  ? _copy_from_user+0x3a/0x60 
[ 1356.327166]  ? simple_transaction_get+0xc4/0xf0 
[ 1356.331732]  ? write_pool_threads+0x230/0x230 [nfsd] 
[ 1356.336763]  nfsctl_transaction_write+0x46/0x80 [nfsd] 
[ 1356.341972]  vfs_write+0xb5/0x280 
[ 1356.345317]  ksys_write+0x5f/0xe0 
[ 1356.348659]  do_syscall_64+0x5c/0x90 
[ 1356.352265]  ? syscall_exit_work+0x11a/0x150 
[ 1356.356568]  ? syscall_exit_to_user_mode+0x12/0x30 
[ 1356.361396]  ? do_syscall_64+0x69/0x90 
[ 1356.365177]  ? do_syscall_64+0x69/0x90 
[ 1356.368957]  ? syscall_exit_work+0x11a/0x150 
[ 1356.373261]  ? syscall_exit_to_user_mode+0x12/0x30 
[ 1356.378086]  ? do_syscall_64+0x69/0x90 
[ 1356.381863]  ? exc_page_fault+0x62/0x150 
[ 1356.385815]  entry_SYSCALL_64_after_hwframe+0x63/0xcd 
[ 1356.390904] RIP: 0033:0x7f296033ebe7 
[ 1356.394507] RSP: 002b:00007ffd338ef548 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 
[ 1356.402128] RAX: ffffffffffffffda RBX: 0000000000000004 RCX: 00007f296033ebe7 
[ 1356.409312] RDX: 0000000000000002 RSI: 000055d77333ac20 RDI: 0000000000000003 
[ 1356.416497] RBP: 0000000000000003 R08: 0000000000000000 R09: 00007ffd338ef3e0 
[ 1356.423680] R10: 0000000000000000 R11: 0000000000000246 R12: 000055d77333ac20 
[ 1356.430864] R13: 00007f29604226c0 R14: 00007ffd338ef610 R15: 00000000ffffffff 
[ 1356.438050]  </TASK> 
[-- MARK -- Mon Jan  9 22:00:00 2023] 
[ 1478.935485] INFO: task nfsd:41446 blocked for more than 368 seconds. 
[ 1478.941894]       Not tainted 5.14.0-231.1852_742217007.el9.x86_64 #1 
[ 1478.948380] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1478.956265] task:nfsd            state:D stack:    0 pid:41446 ppid:     2 flags:0x00004000 
[ 1478.964677] Call Trace: 
[ 1478.967143]  <TASK> 
[ 1478.969260]  __schedule+0x248/0x620 
[ 1478.972782]  schedule+0x5a/0xc0 
[ 1478.975952]  schedule_timeout+0x11d/0x160 
[ 1478.979995]  __wait_for_common+0x93/0x1d0 
[ 1478.984037]  ? usleep_range_state+0x90/0x90 
[ 1478.988256]  svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma] 
[ 1478.994068]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1478.999096]  svc_handle_xprt+0x2ea/0x370 [sunrpc] 
[ 1479.003897]  svc_recv+0xab/0x1b0 [sunrpc] 
[ 1479.007983]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1479.013107]  nfsd+0xc3/0x190 [nfsd] 
[ 1479.016657]  kthread+0xd9/0x100 
[ 1479.019830]  ? kthread_complete_and_exit+0x20/0x20 
[ 1479.024659]  ret_from_fork+0x22/0x30 
[ 1479.028269]  </TASK> 
[ 1479.030476] INFO: task nfsd:41449 blocked for more than 368 seconds. 
[ 1479.036878]       Not tainted 5.14.0-231.1852_742217007.el9.x86_64 #1 
[ 1479.043365] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1479.051250] task:nfsd            state:D stack:    0 pid:41449 ppid:     2 flags:0x00004000 
[ 1479.059664] Call Trace: 
[ 1479.062133]  <TASK> 
[ 1479.064253]  __schedule+0x248/0x620 
[ 1479.067775]  schedule+0x5a/0xc0 
[ 1479.070944]  schedule_timeout+0x11d/0x160 
[ 1479.074987]  __wait_for_common+0x93/0x1d0 
[ 1479.079027]  ? usleep_range_state+0x90/0x90 
[ 1479.083246]  svc_rdma_process_read_list+0x103/0x2d0 [rpcrdma] 
[ 1479.089053]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1479.094073]  svc_handle_xprt+0x2ea/0x370 [sunrpc] 
[ 1479.098855]  svc_recv+0xab/0x1b0 [sunrpc] 
[ 1479.102936]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1479.108057]  nfsd+0xc3/0x190 [nfsd] 
[ 1479.111608]  kthread+0xd9/0x100 
[ 1479.114777]  ? kthread_complete_and_exit+0x20/0x20 
[ 1479.119605]  ret_from_fork+0x22/0x30 
[ 1479.123213]  </TASK> 
[ 1479.125441] INFO: task rpc.nfsd:41699 blocked for more than 368 seconds. 
[ 1479.132195]       Not tainted 5.14.0-231.1852_742217007.el9.x86_64 #1 
[ 1479.138679] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1479.146561] task:rpc.nfsd        state:D stack:    0 pid:41699 ppid:     1 flags:0x00000006 
[ 1479.154971] Call Trace: 
[ 1479.157436]  <TASK> 
[ 1479.159553]  __schedule+0x248/0x620 
[ 1479.163073]  schedule+0x5a/0xc0 
[ 1479.166238]  schedule_timeout+0x11d/0x160 
[ 1479.170278]  ? default_send_IPI_single_phys+0x26/0x40 
[ 1479.175368]  ? ttwu_queue_wakelist+0xf4/0x110 
[ 1479.179760]  __wait_for_common+0x93/0x1d0 
[ 1479.183797]  ? usleep_range_state+0x90/0x90 
[ 1479.188010]  kthread_stop+0x62/0x170 
[ 1479.191614]  svc_set_num_threads+0x10d/0x1a0 [sunrpc] 
[ 1479.196745]  nfsd_svc+0x123/0x190 [nfsd] 
[ 1479.200731]  write_threads+0x95/0x100 [nfsd] 
[ 1479.205065]  ? _copy_from_user+0x3a/0x60 
[ 1479.209020]  ? simple_transaction_get+0xc4/0xf0 
[ 1479.213586]  ? write_pool_threads+0x230/0x230 [nfsd] 
[ 1479.218620]  nfsctl_transaction_write+0x46/0x80 [nfsd] 
[ 1479.223826]  vfs_write+0xb5/0x280 
[ 1479.227169]  ksys_write+0x5f/0xe0 
[ 1479.230512]  do_syscall_64+0x5c/0x90 
[ 1479.234118]  ? syscall_exit_work+0x11a/0x150 
[ 1479.238419]  ? syscall_exit_to_user_mode+0x12/0x30 
[ 1479.243246]  ? do_syscall_64+0x69/0x90 
[ 1479.247023]  ? do_syscall_64+0x69/0x90 
[ 1479.250799]  ? syscall_exit_work+0x11a/0x150 
[ 1479.255101]  ? syscall_exit_to_user_mode+0x12/0x30 
[ 1479.259926]  ? do_syscall_64+0x69/0x90 
[ 1479.263702]  ? exc_page_fault+0x62/0x150 
[ 1479.267656]  entry_SYSCALL_64_after_hwframe+0x63/0xcd 
[ 1479.272745] RIP: 0033:0x7f296033ebe7 
[ 1479.276348] RSP: 002b:00007ffd338ef548 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 
[ 1479.283972] RAX: ffffffffffffffda RBX: 0000000000000004 RCX: 00007f296033ebe7 
[ 1479.291155] RDX: 0000000000000002 RSI: 000055d77333ac20 RDI: 0000000000000003 
[ 1479.298339] RBP: 0000000000000003 R08: 0000000000000000 R09: 00007ffd338ef3e0 
[ 1479.305521] R10: 0000000000000000 R11: 0000000000000246 R12: 000055d77333ac20 
[ 1479.312706] R13: 00007f29604226c0 R14: 00007ffd338ef610 R15: 00000000ffffffff 
[ 1479.319891]  </TASK> 
[ 1479.322095] INFO: task exportfs:42251 blocked for more than 122 seconds. 
[ 1479.328843]       Not tainted 5.14.0-231.1852_742217007.el9.x86_64 #1 
[ 1479.335326] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1479.343209] task:exportfs        state:D stack:    0 pid:42251 ppid:     1 flags:0x00000006 
[ 1479.351619] Call Trace: 
[ 1479.354082]  <TASK> 
[ 1479.356197]  __schedule+0x248/0x620 
[ 1479.359714]  schedule+0x5a/0xc0 
[ 1479.362880]  schedule_preempt_disabled+0x11/0x20 
[ 1479.367529]  __mutex_lock.constprop.0+0x2a1/0x430 
[ 1479.372267]  expkey_flush+0x11/0x40 [nfsd] 
[ 1479.376429]  write_flush.constprop.0+0xf7/0x140 [sunrpc] 
[ 1479.381825]  ? selinux_file_permission+0x108/0x150 
[ 1479.386652]  proc_reg_write+0x55/0xa0 
[ 1479.390347]  vfs_write+0xb5/0x280 
[ 1479.393691]  ksys_write+0x5f/0xe0 
[ 1479.397033]  do_syscall_64+0x5c/0x90 
[ 1479.400635]  ? syscall_exit_work+0x11a/0x150 
[ 1479.404935]  ? syscall_exit_to_user_mode+0x12/0x30 
[ 1479.409758]  ? do_syscall_64+0x69/0x90 
[ 1479.413537]  ? do_syscall_64+0x69/0x90 
[ 1479.417318]  ? do_syscall_64+0x69/0x90 
[ 1479.421095]  entry_SYSCALL_64_after_hwframe+0x63/0xcd 
[ 1479.426181] RIP: 0033:0x7f2052d3ebe7 
[ 1479.429782] RSP: 002b:00007ffe8bb63b68 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 
[ 1479.437403] RAX: ffffffffffffffda RBX: 0000560c57c8fa50 RCX: 00007f2052d3ebe7 
[ 1479.444585] RDX: 000000000000000b RSI: 00007ffe8bb63b70 RDI: 0000000000000004 
[ 1479.451770] RBP: 00007ffe8bb63b90 R08: 0000000000000000 R09: 00007f2052db14e0 
[ 1479.458952] R10: 0000000000000000 R11: 0000000000000246 R12: 00007ffe8bb63b70 
[ 1479.466134] R13: 000000000000000b R14: 0000560c57c8c526 R15: 0000000000000004 
[ 1479.473318]  </TASK> 
[-- MARK -- Mon Jan  9 22:05:00 2023] 
[-- MARK -- Mon Jan  9 22:10:00 2023] 
[-- MARK -- Mon Jan  9 22:15:00 2023] 
[-- MARK -- Mon Jan  9 22:20:00 2023] 
[-- MARK -- Mon Jan  9 22:25:00 2023] 
[-- MARK -- Mon Jan  9 22:30:00 2023] 
[-- MARK -- Mon Jan  9 22:35:00 2023] 
[-- MARK -- Mon Jan  9 22:40:00 2023]