[  216.499860] Running test [R:13122523 T:7 - RDMA Sanity Test - RoCE - Server - Kernel: 5.14.0-217.rt14.218.1773_725926211.el9.x86_64] 
[-- MARK -- Sun Dec 18 16:50:00 2022] 
[  533.904668] systemd-rc-local-generator[29919]: /etc/rc.d/rc.local is not marked executable, skipping. 
[  538.464739] systemd-rc-local-generator[29987]: /etc/rc.d/rc.local is not marked executable, skipping. 
[  543.135396] systemd-rc-local-generator[30166]: /etc/rc.d/rc.local is not marked executable, skipping. 
[  689.995296] systemd-rc-local-generator[38682]: /etc/rc.d/rc.local is not marked executable, skipping. 
[-- MARK -- Sun Dec 18 16:55:00 2022] 
[  834.458222] XFS (sda4): Unmounting Filesystem 
[  907.254542] XFS (sda4): Mounting V5 Filesystem 
[  907.443756] XFS (sda4): Ending clean mount 
[  908.924800] NFSD: Using UMH upcall client tracking operations. 
[  908.924805] NFSD: starting 90-second grace period (net f0000000) 
[-- MARK -- Sun Dec 18 17:00:00 2022] 
[ 1110.423722] INFO: task nfsd:39901 blocked for more than 122 seconds. 
[ 1110.430123]       Not tainted 5.14.0-217.rt14.218.1773_725926211.el9.x86_64 #1 
[ 1110.437391] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1110.445266] task:nfsd            state:D stack:    0 pid:39901 ppid:     2 flags:0x00004000 
[ 1110.453675] Call Trace: 
[ 1110.456137]  <TASK> 
[ 1110.458251]  __schedule+0x25b/0x660 
[ 1110.461768]  schedule+0x5d/0xd0 
[ 1110.464929]  schedule_timeout+0x151/0x190 
[ 1110.468969]  ? preempt_count_add+0x70/0xa0 
[ 1110.473095]  __wait_for_common+0x8e/0x1c0 
[ 1110.477132]  ? usleep_range_state+0x90/0x90 
[ 1110.481347]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1110.487152]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1110.492165]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1110.496943]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1110.501011]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1110.506122]  nfsd+0xc3/0x190 [nfsd] 
[ 1110.509659]  kthread+0x106/0x130 
[ 1110.512909]  ? kthread_complete_and_exit+0x20/0x20 
[ 1110.517732]  ret_from_fork+0x22/0x30 
[ 1110.521336]  </TASK> 
[ 1110.523542] INFO: task nfsd:39903 blocked for more than 122 seconds. 
[ 1110.529934]       Not tainted 5.14.0-217.rt14.218.1773_725926211.el9.x86_64 #1 
[ 1110.537199] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1110.545075] task:nfsd            state:D stack:    0 pid:39903 ppid:     2 flags:0x00004000 
[ 1110.553485] Call Trace: 
[ 1110.555945]  <TASK> 
[ 1110.558057]  __schedule+0x25b/0x660 
[ 1110.561572]  schedule+0x5d/0xd0 
[ 1110.564735]  schedule_timeout+0x151/0x190 
[ 1110.568771]  ? preempt_count_add+0x70/0xa0 
[ 1110.572896]  __wait_for_common+0x8e/0x1c0 
[ 1110.576932]  ? usleep_range_state+0x90/0x90 
[ 1110.581142]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1110.586938]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1110.591947]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1110.596714]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1110.600780]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1110.605889]  nfsd+0xc3/0x190 [nfsd] 
[ 1110.609424]  kthread+0x106/0x130 
[ 1110.612675]  ? kthread_complete_and_exit+0x20/0x20 
[ 1110.617497]  ret_from_fork+0x22/0x30 
[ 1110.621097]  </TASK> 
[ 1110.623299] INFO: task nfsd:39905 blocked for more than 123 seconds. 
[ 1110.629692]       Not tainted 5.14.0-217.rt14.218.1773_725926211.el9.x86_64 #1 
[ 1110.636958] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1110.644837] task:nfsd            state:D stack:    0 pid:39905 ppid:     2 flags:0x00004000 
[ 1110.653241] Call Trace: 
[ 1110.655704]  <TASK> 
[ 1110.657817]  __schedule+0x25b/0x660 
[ 1110.661330]  schedule+0x5d/0xd0 
[ 1110.664493]  schedule_timeout+0x151/0x190 
[ 1110.668533]  ? preempt_count_add+0x70/0xa0 
[ 1110.672657]  __wait_for_common+0x8e/0x1c0 
[ 1110.676693]  ? usleep_range_state+0x90/0x90 
[ 1110.680904]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1110.686703]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1110.691717]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1110.696487]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1110.700555]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1110.705663]  nfsd+0xc3/0x190 [nfsd] 
[ 1110.709197]  kthread+0x106/0x130 
[ 1110.712446]  ? kthread_complete_and_exit+0x20/0x20 
[ 1110.717267]  ret_from_fork+0x22/0x30 
[ 1110.720869]  </TASK> 
[ 1110.723069] INFO: task nfsd:39907 blocked for more than 123 seconds. 
[ 1110.729464]       Not tainted 5.14.0-217.rt14.218.1773_725926211.el9.x86_64 #1 
[ 1110.736728] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1110.744604] task:nfsd            state:D stack:    0 pid:39907 ppid:     2 flags:0x00004000 
[ 1110.753008] Call Trace: 
[ 1110.755471]  <TASK> 
[ 1110.757587]  __schedule+0x25b/0x660 
[ 1110.761099]  schedule+0x5d/0xd0 
[ 1110.764260]  schedule_timeout+0x151/0x190 
[ 1110.768297]  ? preempt_count_add+0x70/0xa0 
[ 1110.772420]  __wait_for_common+0x8e/0x1c0 
[ 1110.776455]  ? usleep_range_state+0x90/0x90 
[ 1110.780667]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1110.786467]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1110.791479]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1110.796245]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1110.800308]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1110.805414]  nfsd+0xc3/0x190 [nfsd] 
[ 1110.808955]  kthread+0x106/0x130 
[ 1110.812204]  ? kthread_complete_and_exit+0x20/0x20 
[ 1110.817028]  ret_from_fork+0x22/0x30 
[ 1110.820628]  </TASK> 
[ 1110.822830] INFO: task rpc.nfsd:40048 blocked for more than 123 seconds. 
[ 1110.829572]       Not tainted 5.14.0-217.rt14.218.1773_725926211.el9.x86_64 #1 
[ 1110.836838] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1110.844715] task:rpc.nfsd        state:D stack:    0 pid:40048 ppid:     1 flags:0x00000006 
[ 1110.853121] Call Trace: 
[ 1110.855583]  <TASK> 
[ 1110.857698]  __schedule+0x25b/0x660 
[ 1110.861213]  schedule+0x5d/0xd0 
[ 1110.864376]  schedule_timeout+0x151/0x190 
[ 1110.868411]  ? default_send_IPI_single_phys+0x26/0x40 
[ 1110.873499]  ? preempt_count_add+0x70/0xa0 
[ 1110.877622]  __wait_for_common+0x8e/0x1c0 
[ 1110.881659]  ? usleep_range_state+0x90/0x90 
[ 1110.885871]  kthread_stop+0x62/0x1d0 
[ 1110.889470]  svc_set_num_threads+0x10d/0x200 [sunrpc] 
[ 1110.894588]  nfsd_svc+0x123/0x190 [nfsd] 
[ 1110.898560]  write_threads+0x95/0x100 [nfsd] 
[ 1110.902882]  ? _copy_from_user+0x3a/0x60 
[ 1110.906835]  ? simple_transaction_get+0xc4/0xf0 
[ 1110.911396]  ? write_pool_threads+0x230/0x230 [nfsd] 
[ 1110.916415]  nfsctl_transaction_write+0x46/0x80 [nfsd] 
[ 1110.921611]  vfs_write+0xb5/0x2e0 
[ 1110.924949]  ksys_write+0x62/0xe0 
[ 1110.928286]  do_syscall_64+0x5c/0x90 
[ 1110.931889]  ? migrate_enable+0xd5/0x150 
[ 1110.935837]  ? do_sys_openat2+0x85/0x160 
[ 1110.939790]  ? syscall_exit_to_user_mode+0x12/0x30 
[ 1110.944615]  ? do_syscall_64+0x69/0x90 
[ 1110.948393]  ? do_syscall_64+0x69/0x90 
[ 1110.952168]  entry_SYSCALL_64_after_hwframe+0x63/0xcd 
[ 1110.957253] RIP: 0033:0x7f7bf1ef2be7 
[ 1110.960851] RSP: 002b:00007ffe6ab1dae8 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 
[ 1110.968468] RAX: ffffffffffffffda RBX: 0000000000000004 RCX: 00007f7bf1ef2be7 
[ 1110.975647] RDX: 0000000000000002 RSI: 0000562393ab1c20 RDI: 0000000000000003 
[ 1110.982826] RBP: 0000000000000003 R08: 0000000000000000 R09: 00007ffe6ab1d980 
[ 1110.990006] R10: 0000000000000000 R11: 0000000000000246 R12: 0000562393ab1c20 
[ 1110.997186] R13: 00007f7bf1db16c0 R14: 00007ffe6ab1dbb0 R15: 00000000ffffffff 
[ 1111.004371]  </TASK> 
[ 1233.305263] INFO: task nfsd:39901 blocked for more than 245 seconds. 
[ 1233.311661]       Not tainted 5.14.0-217.rt14.218.1773_725926211.el9.x86_64 #1 
[ 1233.318930] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1233.326815] task:nfsd            state:D stack:    0 pid:39901 ppid:     2 flags:0x00004000 
[ 1233.335219] Call Trace: 
[ 1233.337681]  <TASK> 
[ 1233.339794]  __schedule+0x25b/0x660 
[ 1233.343308]  schedule+0x5d/0xd0 
[ 1233.346471]  schedule_timeout+0x151/0x190 
[ 1233.350508]  ? preempt_count_add+0x70/0xa0 
[ 1233.354630]  __wait_for_common+0x8e/0x1c0 
[ 1233.358665]  ? usleep_range_state+0x90/0x90 
[ 1233.362880]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1233.368680]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1233.373692]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1233.378468]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1233.382534]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1233.387646]  nfsd+0xc3/0x190 [nfsd] 
[ 1233.391187]  kthread+0x106/0x130 
[ 1233.394436]  ? kthread_complete_and_exit+0x20/0x20 
[ 1233.399257]  ret_from_fork+0x22/0x30 
[ 1233.402858]  </TASK> 
[ 1233.405064] INFO: task nfsd:39903 blocked for more than 245 seconds. 
[ 1233.411458]       Not tainted 5.14.0-217.rt14.218.1773_725926211.el9.x86_64 #1 
[ 1233.418726] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1233.426604] task:nfsd            state:D stack:    0 pid:39903 ppid:     2 flags:0x00004000 
[ 1233.435008] Call Trace: 
[ 1233.437469]  <TASK> 
[ 1233.439580]  __schedule+0x25b/0x660 
[ 1233.443092]  schedule+0x5d/0xd0 
[ 1233.446254]  schedule_timeout+0x151/0x190 
[ 1233.450289]  ? preempt_count_add+0x70/0xa0 
[ 1233.454417]  __wait_for_common+0x8e/0x1c0 
[ 1233.458454]  ? usleep_range_state+0x90/0x90 
[ 1233.462665]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1233.468462]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1233.473471]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1233.478239]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1233.482303]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1233.487410]  nfsd+0xc3/0x190 [nfsd] 
[ 1233.490946]  kthread+0x106/0x130 
[ 1233.494194]  ? kthread_complete_and_exit+0x20/0x20 
[ 1233.499017]  ret_from_fork+0x22/0x30 
[ 1233.502617]  </TASK> 
[ 1233.504818] INFO: task nfsd:39905 blocked for more than 245 seconds. 
[ 1233.511211]       Not tainted 5.14.0-217.rt14.218.1773_725926211.el9.x86_64 #1 
[ 1233.518482] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1233.526360] task:nfsd            state:D stack:    0 pid:39905 ppid:     2 flags:0x00004000 
[ 1233.534765] Call Trace: 
[ 1233.537227]  <TASK> 
[ 1233.539340]  __schedule+0x25b/0x660 
[ 1233.542853]  schedule+0x5d/0xd0 
[ 1233.546016]  schedule_timeout+0x151/0x190 
[ 1233.550050]  ? preempt_count_add+0x70/0xa0 
[ 1233.554186]  __wait_for_common+0x8e/0x1c0 
[ 1233.558223]  ? usleep_range_state+0x90/0x90 
[ 1233.562436]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1233.568232]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1233.573242]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1233.578007]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1233.582077]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1233.587185]  nfsd+0xc3/0x190 [nfsd] 
[ 1233.590719]  kthread+0x106/0x130 
[ 1233.593969]  ? kthread_complete_and_exit+0x20/0x20 
[ 1233.598789]  ret_from_fork+0x22/0x30 
[ 1233.602390]  </TASK> 
[ 1233.604589] INFO: task nfsd:39907 blocked for more than 246 seconds. 
[ 1233.610982]       Not tainted 5.14.0-217.rt14.218.1773_725926211.el9.x86_64 #1 
[ 1233.618247] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1233.626124] task:nfsd            state:D stack:    0 pid:39907 ppid:     2 flags:0x00004000 
[ 1233.634526] Call Trace: 
[ 1233.636986]  <TASK> 
[ 1233.639097]  __schedule+0x25b/0x660 
[ 1233.642610]  schedule+0x5d/0xd0 
[ 1233.645776]  schedule_timeout+0x151/0x190 
[ 1233.649811]  ? preempt_count_add+0x70/0xa0 
[ 1233.653933]  __wait_for_common+0x8e/0x1c0 
[ 1233.657967]  ? usleep_range_state+0x90/0x90 
[ 1233.662180]  svc_rdma_process_read_list+0x103/0x300 [rpcrdma] 
[ 1233.667974]  svc_rdma_recvfrom+0x217/0x470 [rpcrdma] 
[ 1233.672983]  svc_handle_xprt+0x350/0x400 [sunrpc] 
[ 1233.677747]  svc_recv+0xa1/0x1d0 [sunrpc] 
[ 1233.681810]  ? nfsd_shutdown_threads+0xa0/0xa0 [nfsd] 
[ 1233.686917]  nfsd+0xc3/0x190 [nfsd] 
[ 1233.690450]  kthread+0x106/0x130 
[ 1233.693698]  ? kthread_complete_and_exit+0x20/0x20 
[ 1233.698520]  ret_from_fork+0x22/0x30 
[ 1233.702120]  </TASK> 
[ 1233.704323] INFO: task rpc.nfsd:40048 blocked for more than 246 seconds. 
[ 1233.711075]       Not tainted 5.14.0-217.rt14.218.1773_725926211.el9.x86_64 #1 
[ 1233.718340] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. 
[ 1233.726217] task:rpc.nfsd        state:D stack:    0 pid:40048 ppid:     1 flags:0x00000006 
[ 1233.734620] Call Trace: 
[ 1233.737082]  <TASK> 
[ 1233.739194]  __schedule+0x25b/0x660 
[ 1233.742707]  schedule+0x5d/0xd0 
[ 1233.745869]  schedule_timeout+0x151/0x190 
[ 1233.749902]  ? default_send_IPI_single_phys+0x26/0x40 
[ 1233.754988]  ? preempt_count_add+0x70/0xa0 
[ 1233.759113]  __wait_for_common+0x8e/0x1c0 
[ 1233.763150]  ? usleep_range_state+0x90/0x90 
[ 1233.767363]  kthread_stop+0x62/0x1d0 
[ 1233.770963]  svc_set_num_threads+0x10d/0x200 [sunrpc] 
[ 1233.776086]  nfsd_svc+0x123/0x190 [nfsd] 
[ 1233.780070]  write_threads+0x95/0x100 [nfsd] 
[ 1233.784391]  ? _copy_from_user+0x3a/0x60 
[ 1233.788338]  ? simple_transaction_get+0xc4/0xf0 
[ 1233.792898]  ? write_pool_threads+0x230/0x230 [nfsd] 
[ 1233.797917]  nfsctl_transaction_write+0x46/0x80 [nfsd] 
[ 1233.803113]  vfs_write+0xb5/0x2e0 
[ 1233.806448]  ksys_write+0x62/0xe0 
[ 1233.809783]  do_syscall_64+0x5c/0x90 
[ 1233.813384]  ? migrate_enable+0xd5/0x150 
[ 1233.817334]  ? do_sys_openat2+0x85/0x160 
[ 1233.821284]  ? syscall_exit_to_user_mode+0x12/0x30 
[ 1233.826104]  ? do_syscall_64+0x69/0x90 
[ 1233.829879]  ? do_syscall_64+0x69/0x90 
[ 1233.833654]  entry_SYSCALL_64_after_hwframe+0x63/0xcd 
[ 1233.838742] RIP: 0033:0x7f7bf1ef2be7 
[ 1233.842339] RSP: 002b:00007ffe6ab1dae8 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 
[ 1233.849957] RAX: ffffffffffffffda RBX: 0000000000000004 RCX: 00007f7bf1ef2be7 
[ 1233.857136] RDX: 0000000000000002 RSI: 0000562393ab1c20 RDI: 0000000000000003 
[ 1233.864315] RBP: 0000000000000003 R08: 0000000000000000 R09: 00007ffe6ab1d980 
[ 1233.871495] R10: 0000000000000000 R11: 0000000000000246 R12: 0000562393ab1c20 
[ 1233.878676] R13: 00007f7bf1db16c0 R14: 00007ffe6ab1dbb0 R15: 00000000ffffffff 
[ 1233.885859]  </TASK> 
[-- MARK -- Sun Dec 18 17:05:00 2022] 
[-- MARK -- Sun Dec 18 17:10:00 2022] 
[-- MARK -- Sun Dec 18 17:15:00 2022] 
[-- MARK -- Sun Dec 18 17:20:00 2022] 
[-- MARK -- Sun Dec 18 17:25:00 2022] 
[-- MARK -- Sun Dec 18 17:30:00 2022] 
[-- MARK -- Sun Dec 18 17:35:00 2022] 
[-- MARK -- Sun Dec 18 17:40:00 2022] 
[-- MARK -- Sun Dec 18 17:45:00 2022]