+ [23-01-15 09:19:53] '[' multi == multi ']' + [23-01-15 09:19:53] echo 'Clients: rdma-qe-30.rdma.lab.eng.rdu2.redhat.com' Clients: rdma-qe-30.rdma.lab.eng.rdu2.redhat.com + [23-01-15 09:19:53] echo 'Servers: rdma-qe-31.rdma.lab.eng.rdu2.redhat.com' Servers: rdma-qe-31.rdma.lab.eng.rdu2.redhat.com + [23-01-15 09:19:53] RQA_system_info_for_debug + [23-01-15 09:19:53] grep -i distro /etc/motd + [23-01-15 09:19:53] tr -d ' ' DISTRO=CentOS-Stream-9-20230109.0 + [23-01-15 09:19:53] cat /etc/redhat-release CentOS Stream release 9 + [23-01-15 09:19:53] uname -a Linux rdma-qe-31.rdma.lab.eng.rdu2.redhat.com 5.14.0-235.rt14.236.1639_747671795.el9.x86_64 #1 SMP PREEMPT_RT Sun Jan 15 12:47:38 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux + [23-01-15 09:19:53] cat /proc/cmdline BOOT_IMAGE=(hd0,gpt2)/vmlinuz-5.14.0-235.rt14.236.1639_747671795.el9.x86_64 root=UUID=e6dfca68-68e9-4ada-872e-5d60193d151c ro efi=runtime resume=UUID=b1f3698a-6dd5-46a4-827b-26a8975186b6 console=ttyS0,115200n81 crashkernel=1G-4G:384M,4G-16G:512M,16G-64G:1G,64G-128G:2G,128G-:4G + [23-01-15 09:19:53] rpm -q rdma-core linux-firmware rdma-core-41.0-3.el9.x86_64 linux-firmware-20221214-129.el9.noarch + [23-01-15 09:19:53] tail /sys/class/infiniband/bnxt_re0/fw_ver /sys/class/infiniband/bnxt_re1/fw_ver ==> /sys/class/infiniband/bnxt_re0/fw_ver <== 214.0.182.0 ==> /sys/class/infiniband/bnxt_re1/fw_ver <== 214.0.182.0 + [23-01-15 09:19:53] lspci + [23-01-15 09:19:53] grep -i -e ethernet -e infiniband -e omni -e ConnectX 44:00.0 Ethernet controller: Broadcom Inc. and subsidiaries BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller 44:00.1 Ethernet controller: Broadcom Inc. and subsidiaries BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller c5:00.0 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01) c5:00.1 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01) c5:00.2 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01) c5:00.3 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01) + [23-01-15 09:19:54] lscpu Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Address sizes: 48 bits physical, 48 bits virtual Byte Order: Little Endian CPU(s): 48 On-line CPU(s) list: 0-47 Vendor ID: AuthenticAMD BIOS Vendor ID: Advanced Micro Devices, Inc. Model name: AMD EPYC 7402P 24-Core Processor BIOS Model name: AMD EPYC 7402P 24-Core Processor CPU family: 23 Model: 49 Thread(s) per core: 2 Core(s) per socket: 24 Socket(s): 1 Stepping: 0 Frequency boost: enabled CPU max MHz: 2800.0000 CPU min MHz: 1500.0000 BogoMIPS: 5589.37 Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca Virtualization: AMD-V L1d cache: 768 KiB (24 instances) L1i cache: 768 KiB (24 instances) L2 cache: 12 MiB (24 instances) L3 cache: 128 MiB (8 instances) NUMA node(s): 1 NUMA node0 CPU(s): 0-47 Vulnerability Itlb multihit: Not affected Vulnerability L1tf: Not affected Vulnerability Mds: Not affected Vulnerability Meltdown: Not affected Vulnerability Mmio stale data: Not affected Vulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization Vulnerability Spectre v2: Mitigation; Retpolines, IBPB conditional, STIBP always-on, RSB filling, PBRSB-eIBRS Not affected Vulnerability Srbds: Not affected Vulnerability Tsx async abort: Not affected + [23-01-15 09:19:54] ibstat CA 'bnxt_re0' CA type: Broadcom NetXtreme-C/E RoCE Driver HCA Number of ports: 1 Firmware version: 214.0.182.0 Hardware version: 0x14e4 Node GUID: 0x020af7fffe97b162 System image GUID: 0x020af7fffe97b162 Port 1: State: Active Physical state: LinkUp Rate: 25 Base lid: 0 LMC: 0 SM lid: 0 Capability mask: 0x001d0000 Port GUID: 0x020af7fffe97b162 Link layer: Ethernet CA 'bnxt_re1' CA type: Broadcom NetXtreme-C/E RoCE Driver HCA Number of ports: 1 Firmware version: 214.0.182.0 Hardware version: 0x14e4 Node GUID: 0x020af7fffe97b163 System image GUID: 0x020af7fffe97b163 Port 1: State: Down Physical state: Disabled Rate: 2.5 Base lid: 0 LMC: 0 SM lid: 0 Capability mask: 0x001d0000 Port GUID: 0x020af7fffe97b163 Link layer: Ethernet + [23-01-15 09:19:56] ibstatus Infiniband device 'bnxt_re0' port 1 status: default gid: fe80:0000:0000:0000:020a:f7ff:fe97:b162 base lid: 0x0 sm lid: 0x0 state: 4: ACTIVE phys state: 5: LinkUp rate: 25 Gb/sec (1X EDR) link_layer: Ethernet Infiniband device 'bnxt_re1' port 1 status: default gid: fe80:0000:0000:0000:020a:f7ff:fe97:b163 base lid: 0x0 sm lid: 0x0 state: 1: DOWN phys state: 3: Disabled rate: 2.5 Gb/sec (1X SDR) link_layer: Ethernet + [23-01-15 09:19:56] /usr/bin/ibv_devinfo -v hca_id: bnxt_re0 transport: InfiniBand (0) fw_ver: 214.0.182.0 node_guid: 020a:f7ff:fe97:b162 sys_image_guid: 020a:f7ff:fe97:b162 vendor_id: 0x14e4 vendor_part_id: 5847 hw_ver: 0x1402 phys_port_cnt: 1 max_mr_size: 0x8000000000 page_size_cap: 0x201000 max_qp: 65408 max_qp_wr: 65406 device_cap_flags: 0x01225d81 RESIZE_MAX_WR CURR_QP_STATE_MOD SHUTDOWN_PORT PORT_ACTIVE_EVENT SYS_IMAGE_GUID RC_RNR_NAK_GEN N_NOTIFY_CQ MEM_WINDOW MEM_MGT_EXTENSIONS MEM_WINDOW_TYPE_2B max_sge: 6 max_sge_rd: 6 max_cq: 65535 max_cqe: 1048575 max_mr: 262143 max_pd: 65536 max_qp_rd_atom: 16 max_ee_rd_atom: 0 max_res_rd_atom: 0 max_qp_init_rd_atom: 126 max_ee_init_rd_atom: 0 atomic_cap: ATOMIC_NONE (0) max_ee: 0 max_rdd: 0 max_mw: 524288 max_raw_ipv6_qp: 0 max_raw_ethy_qp: 524288 max_mcast_grp: 0 max_mcast_qp_attach: 0 max_total_mcast_qp_attach: 0 max_ah: 524288 max_fmr: 0 max_srq: 32767 max_srq_wr: 1048574 max_srq_sge: 6 max_pkeys: 1 local_ca_ack_delay: 16 general_odp_caps: rc_odp_caps: NO SUPPORT uc_odp_caps: NO SUPPORT ud_odp_caps: NO SUPPORT xrc_odp_caps: NO SUPPORT completion_timestamp_mask not supported core clock not supported device_cap_flags_ex: 0x1225D81 tso_caps: max_tso: 0 rss_caps: max_rwq_indirection_tables: 0 max_rwq_indirection_table_size: 0 rx_hash_function: 0x0 rx_hash_fields_mask: 0x0 max_wq_type_rq: 0 packet_pacing_caps: qp_rate_limit_min: 0kbps qp_rate_limit_max: 0kbps tag matching not supported num_comp_vectors: 8 port: 1 state: PORT_ACTIVE (4) max_mtu: 4096 (5) active_mtu: 4096 (5) sm_lid: 0 port_lid: 0 port_lmc: 0x00 link_layer: Ethernet max_msg_sz: 0x40000000 port_cap_flags: 0x041d0000 port_cap_flags2: 0x0000 max_vl_num: 8 (4) bad_pkey_cntr: 0x0 qkey_viol_cntr: 0x0 sm_sl: 0 pkey_tbl_len: 65535 gid_tbl_len: 256 subnet_timeout: 0 init_type_reply: 0 active_width: 1X (1) active_speed: 25.0 Gbps (32) phys_state: LINK_UP (5) GID[ 0]: fe80:0000:0000:0000:020a:f7ff:fe97:b162, RoCE v1 GID[ 1]: fe80::20a:f7ff:fe97:b162, RoCE v2 GID[ 2]: 0000:0000:0000:0000:0000:ffff:ac1f:281f, RoCE v1 GID[ 3]: ::ffff:172.31.40.31, RoCE v2 GID[ 4]: 0000:0000:0000:0000:0000:ffff:ac1f:2b1f, RoCE v1 GID[ 5]: ::ffff:172.31.43.31, RoCE v2 GID[ 6]: fe80:0000:0000:0000:020a:f7ff:fe97:b162, RoCE v1 GID[ 7]: fe80::20a:f7ff:fe97:b162, RoCE v2 GID[ 8]: 0000:0000:0000:0000:0000:ffff:ac1f:2d1f, RoCE v1 GID[ 9]: ::ffff:172.31.45.31, RoCE v2 GID[ 10]: fe80:0000:0000:0000:020a:f7ff:fe97:b162, RoCE v1 GID[ 11]: fe80::20a:f7ff:fe97:b162, RoCE v2 hca_id: bnxt_re1 transport: InfiniBand (0) fw_ver: 214.0.182.0 node_guid: 020a:f7ff:fe97:b163 sys_image_guid: 020a:f7ff:fe97:b163 vendor_id: 0x14e4 vendor_part_id: 5847 hw_ver: 0x1402 phys_port_cnt: 1 max_mr_size: 0x8000000000 page_size_cap: 0x201000 max_qp: 65408 max_qp_wr: 65406 device_cap_flags: 0x01225d81 RESIZE_MAX_WR CURR_QP_STATE_MOD SHUTDOWN_PORT PORT_ACTIVE_EVENT SYS_IMAGE_GUID RC_RNR_NAK_GEN N_NOTIFY_CQ MEM_WINDOW MEM_MGT_EXTENSIONS MEM_WINDOW_TYPE_2B max_sge: 6 max_sge_rd: 6 max_cq: 65535 max_cqe: 1048575 max_mr: 262143 max_pd: 65536 max_qp_rd_atom: 16 max_ee_rd_atom: 0 max_res_rd_atom: 0 max_qp_init_rd_atom: 126 max_ee_init_rd_atom: 0 atomic_cap: ATOMIC_NONE (0) max_ee: 0 max_rdd: 0 max_mw: 524288 max_raw_ipv6_qp: 0 max_raw_ethy_qp: 524288 max_mcast_grp: 0 max_mcast_qp_attach: 0 max_total_mcast_qp_attach: 0 max_ah: 524288 max_fmr: 0 max_srq: 32767 max_srq_wr: 1048574 max_srq_sge: 6 max_pkeys: 1 local_ca_ack_delay: 16 general_odp_caps: rc_odp_caps: NO SUPPORT uc_odp_caps: NO SUPPORT ud_odp_caps: NO SUPPORT xrc_odp_caps: NO SUPPORT completion_timestamp_mask not supported core clock not supported device_cap_flags_ex: 0x1225D81 tso_caps: max_tso: 0 rss_caps: max_rwq_indirection_tables: 0 max_rwq_indirection_table_size: 0 rx_hash_function: 0x0 rx_hash_fields_mask: 0x0 max_wq_type_rq: 0 packet_pacing_caps: qp_rate_limit_min: 0kbps qp_rate_limit_max: 0kbps tag matching not supported num_comp_vectors: 8 port: 1 state: PORT_DOWN (1) max_mtu: 4096 (5) active_mtu: 1024 (3) sm_lid: 0 port_lid: 0 port_lmc: 0x00 link_layer: Ethernet max_msg_sz: 0x40000000 port_cap_flags: 0x041d0000 port_cap_flags2: 0x0000 max_vl_num: 8 (4) bad_pkey_cntr: 0x0 qkey_viol_cntr: 0x0 sm_sl: 0 pkey_tbl_len: 65535 gid_tbl_len: 256 subnet_timeout: 0 init_type_reply: 0 active_width: 1X (1) active_speed: 2.5 Gbps (1) phys_state: DISABLED (3) GID[ 0]: fe80:0000:0000:0000:020a:f7ff:fe97:b163, RoCE v1 GID[ 1]: fe80::20a:f7ff:fe97:b163, RoCE v2 + [23-01-15 09:19:57] iblinkinfo ibwarn: [11628] get_abi_version: can't read ABI version from /sys/class/infiniband_mad/abi_version (No such file or directory): is ib_umad module loaded? ibwarn: [11628] mad_rpc_open_port: can't open UMAD port ((null):0) Failed to open (null) port 0 + [23-01-15 09:19:57] [[ '' == *\o\p\a* ]] + [23-01-15 09:19:57] ip addr show 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: bnxt_roce: mtu 9000 qdisc mq state UP group default qlen 1000 link/ether 00:0a:f7:97:b1:62 brd ff:ff:ff:ff:ff:ff altname enp68s0f0np0 altname ens2f0np0 inet 172.31.40.31/24 brd 172.31.40.255 scope global dynamic noprefixroute bnxt_roce valid_lft 3387sec preferred_lft 3387sec inet6 fe80::20a:f7ff:fe97:b162/64 scope link noprefixroute valid_lft forever preferred_lft forever 3: bnxt_off: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 00:0a:f7:97:b1:63 brd ff:ff:ff:ff:ff:ff altname enp68s0f1np1 altname ens2f1np1 4: lom_1: mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 48:df:37:ad:f6:f0 brd ff:ff:ff:ff:ff:ff altname enp197s0f0 altname eno5 inet 10.1.217.81/24 brd 10.1.217.255 scope global dynamic noprefixroute lom_1 valid_lft 86190sec preferred_lft 86190sec inet6 2620:52:0:1d9:4adf:37ff:fead:f6f0/64 scope global dynamic noprefixroute valid_lft 2591955sec preferred_lft 604755sec inet6 fe80::4adf:37ff:fead:f6f0/64 scope link noprefixroute valid_lft forever preferred_lft forever 5: lom_2: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 48:df:37:ad:f6:f1 brd ff:ff:ff:ff:ff:ff altname enp197s0f1 altname eno6 6: lom_3: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 48:df:37:ad:f6:f2 brd ff:ff:ff:ff:ff:ff altname enp197s0f2 altname eno7 7: lom_4: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 48:df:37:ad:f6:f3 brd ff:ff:ff:ff:ff:ff altname enp197s0f3 altname eno8 8: bnxt_roce.43@bnxt_roce: mtu 9000 qdisc noqueue state UP group default qlen 1000 link/ether 00:0a:f7:97:b1:62 brd ff:ff:ff:ff:ff:ff inet 172.31.43.31/24 brd 172.31.43.255 scope global dynamic noprefixroute bnxt_roce.43 valid_lft 3387sec preferred_lft 3387sec inet6 fe80::20a:f7ff:fe97:b162/64 scope link noprefixroute valid_lft forever preferred_lft forever 9: bnxt_roce.45@bnxt_roce: mtu 9000 qdisc noqueue state UP group default qlen 1000 link/ether 00:0a:f7:97:b1:62 brd ff:ff:ff:ff:ff:ff inet 172.31.45.31/24 brd 172.31.45.255 scope global dynamic noprefixroute bnxt_roce.45 valid_lft 3387sec preferred_lft 3387sec inet6 fe80::20a:f7ff:fe97:b162/64 scope link noprefixroute valid_lft forever preferred_lft forever + [23-01-15 09:19:57] grep -i fedora /etc/redhat-release + [23-01-15 09:19:57] [[ '' == \i\4\0\i\w ]] + [23-01-15 09:19:57] hostname -A + [23-01-15 09:19:57] grep rdma-qe-30 + [23-01-15 09:19:57] hostname -A + [23-01-15 09:19:57] grep rdma-qe-31 + [23-01-15 09:19:57] echo '------- server is ready -----' ------- server is ready ----- + [23-01-15 09:19:57] server + [23-01-15 09:19:57] source ./setup.sh + [23-01-15 09:19:57] server_setup + [23-01-15 09:19:57] is_cki + [23-01-15 09:19:57] typeset -l flag=true + [23-01-15 09:19:57] [[ true == \t\r\u\e ]] + [23-01-15 09:19:57] return 0 + [23-01-15 09:19:57] run_task_env_setup + [23-01-15 09:19:57] echo 'Now start to run task env_setup, please be patient ...' Now start to run task env_setup, please be patient ... + [23-01-15 09:19:57] cd ../../env_setup + [23-01-15 09:19:57] make run make[1]: Entering directory '/mnt/tests/gitlab.cee.redhat.com/kernel-qe/kernel/-/archive/master/kernel-master.zip/infiniband/env_setup' test -x runtest.sh || chmod a+x runtest.sh ./runtest.sh + trap 'exit 1' SIGHUP SIGINT SIGQUIT SIGTERM + which wget + alias + eval declare -f ++ declare -f + /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot wget /usr/bin/wget + source /usr/bin/rhts_environment.sh ++ '[' -z /mnt/testarea/tmp.mSwgTm ']' ++ '[' -z '' ']' +++ uname -i ++ ARCH=x86_64 ++ '[' -z '' ']' +++ cat /etc/redhat-release +++ sed -e 's/\(.*\)release\s\([0-9]*\).*/\1\2/; s/\s//g' ++ FAMILY=CentOSStream9 ++ '[' -h /mnt/testarea/current.log ']' ++ ln -sf /mnt/testarea/tmp.mSwgTm /mnt/testarea/current.log + '[' -f /root/rdma-qa.sh ']' + source /root/rdma-qa.sh ++ PS4='$(date +"+ [%y-%m-%d %H:%M:%S]") ' ++ [23-01-15 09:19:57] export NIC_PREF= ++ [23-01-15 09:19:57] NIC_PREF= ++ [23-01-15 09:19:57] export PERFTEST_FLAGS= ++ [23-01-15 09:19:57] PERFTEST_FLAGS= +++ [23-01-15 09:19:57] hostname -s ++ [23-01-15 09:19:57] export s_hostname=rdma-qe-31 ++ [23-01-15 09:19:57] s_hostname=rdma-qe-31 ++ [23-01-15 09:19:57] RQA_set_pyexec ++ [23-01-15 09:19:57] PYEXEC= ++ [23-01-15 09:19:57] which python ++ [23-01-15 09:19:57] export PYEXEC=python ++ [23-01-15 09:19:57] PYEXEC=python ++ [23-01-15 09:19:57] return 0 +++ [23-01-15 09:19:57] grep -i fedora /etc/redhat-release ++ [23-01-15 09:19:57] [[ -n '' ]] +++ [23-01-15 09:19:57] RQA_get_rhel_major +++ [23-01-15 09:19:57] awk -F . '{print $1}' ++++ [23-01-15 09:19:57] RQA_get_rhel_release ++++ [23-01-15 09:19:57] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [23-01-15 09:19:57] echo 9 ++ [23-01-15 09:19:57] [[ 9 -ge 8 ]] ++ [23-01-15 09:19:57] export 'PKGINSTALL=dnf install -y --setopt=strict=0 --nogpgcheck' ++ [23-01-15 09:19:57] PKGINSTALL='dnf install -y --setopt=strict=0 --nogpgcheck' ++ [23-01-15 09:19:57] export 'PKGREMOVE=dnf remove --noautoremove -y' ++ [23-01-15 09:19:57] PKGREMOVE='dnf remove --noautoremove -y' ++ [23-01-15 09:19:57] [[ rdma-qe-31 == \r\d\m\a\-\d\e\v\-\2\0 ]] ++ [23-01-15 09:19:57] export LATEST_MFT=http://www.mellanox.com/downloads/MFT/mft-4.10.0-104-x86_64-rpm.tgz ++ [23-01-15 09:19:57] LATEST_MFT=http://www.mellanox.com/downloads/MFT/mft-4.10.0-104-x86_64-rpm.tgz ++ [23-01-15 09:19:57] '[' -f /root/known-issues.sh ']' ++ [23-01-15 09:19:57] source /root/known-issues.sh ++ [23-01-15 09:20:01] RQA_load_tech_preview_modules ++ [23-01-15 09:20:01] [[ ! -e /etc/rdma/rdma.conf ]] ++ [23-01-15 09:20:01] return ++ [23-01-15 09:20:01] RQA_fix_uefi_reboot ++ [23-01-15 09:20:01] ls -l /usr/sbin/reboot ++ [23-01-15 09:20:01] grep rhts-reboot ++ [23-01-15 09:20:01] which rhts-reboot ++ [23-01-15 09:20:01] rpm -q rhts-test-env ++ [23-01-15 09:20:01] RQA_fix_ssh_ipv6_auth +++ [23-01-15 09:20:01] RQA_get_rhel_major +++ [23-01-15 09:20:01] awk -F . '{print $1}' ++++ [23-01-15 09:20:01] RQA_get_rhel_release ++++ [23-01-15 09:20:01] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [23-01-15 09:20:01] echo 9 ++ [23-01-15 09:20:01] '[' 9 -ge 7 ']' ++ [23-01-15 09:20:01] local need_restart=0 ++ [23-01-15 09:20:01] grep -E '^UseDNS yes' /etc/ssh/sshd_config ++ [23-01-15 09:20:01] grep -E '^GSSAPIStrictAcceptorCheck yes' /etc/ssh/sshd_config ++ [23-01-15 09:20:01] grep 'AddressFamily inet' /root/.ssh/config ++ [23-01-15 09:20:01] '[' 0 -eq 1 ']' ++ [23-01-15 09:20:01] RQA_fix_mlx5_ib_connected_mode +++ [23-01-15 09:20:01] RQA_get_rhel_major +++ [23-01-15 09:20:01] awk -F . '{print $1}' ++++ [23-01-15 09:20:01] RQA_get_rhel_release ++++ [23-01-15 09:20:01] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [23-01-15 09:20:01] echo 9 ++ [23-01-15 09:20:01] [[ 9 -ge 7 ]] +++ [23-01-15 09:20:01] RQA_get_rhel_minor +++ [23-01-15 09:20:01] awk -F . '{print $2}' ++++ [23-01-15 09:20:01] RQA_get_rhel_release ++++ [23-01-15 09:20:01] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [23-01-15 09:20:01] echo 9 ++ [23-01-15 09:20:01] [[ '' -ge 5 ]] +++ [23-01-15 09:20:01] RQA_get_rhel_major +++ [23-01-15 09:20:01] awk -F . '{print $1}' ++++ [23-01-15 09:20:01] RQA_get_rhel_release ++++ [23-01-15 09:20:01] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [23-01-15 09:20:01] echo 9 ++ [23-01-15 09:20:01] [[ 9 -gt 7 ]] +++ [23-01-15 09:20:01] ls /etc/sysconfig/network-scripts/ +++ [23-01-15 09:20:01] grep mlx5 +++ [23-01-15 09:20:01] grep ib +++ [23-01-15 09:20:01] grep -v '~' ++ [23-01-15 09:20:01] RQA_nfs_rhel8_temp_fix +++ [23-01-15 09:20:01] RQA_get_rhel_major +++ [23-01-15 09:20:01] awk -F . '{print $1}' ++++ [23-01-15 09:20:01] RQA_get_rhel_release ++++ [23-01-15 09:20:01] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [23-01-15 09:20:01] echo 9 ++ [23-01-15 09:20:01] (( 9 == 8 )) ++ [23-01-15 09:20:01] source /root/.bashrc ++ [23-01-15 09:20:02] [[ '' == \i\4\0\i\w ]] + [23-01-15 09:20:02] export BASHRC_FILE=/root/.bashrc + [23-01-15 09:20:02] BASHRC_FILE=/root/.bashrc + [23-01-15 09:20:02] [[ rdma-qe-31 == \r\d\m\a\-\p\e\r\f\-\0\2 ]] + [23-01-15 09:20:02] [[ rdma-qe-31 == \r\d\m\a\-\p\e\r\f\-\0\3 ]] + [23-01-15 09:20:02] export ENV_DRIVER= + [23-01-15 09:20:02] ENV_DRIVER= + [23-01-15 09:20:02] export ENV_NETWORK=roce + [23-01-15 09:20:02] ENV_NETWORK=roce ++ [23-01-15 09:20:02] hostname -s + [23-01-15 09:20:02] export host_json=./json/rdma-qe-31.json + [23-01-15 09:20:02] host_json=./json/rdma-qe-31.json + [23-01-15 09:20:02] [[ rdma-qe-31.rdma.lab.eng.rdu2.redhat.com == *\r\d\m\a\-\d\e\v\-\3\0* ]] + [23-01-15 09:20:02] [[ rdma-qe-31.rdma.lab.eng.rdu2.redhat.com == *\r\d\m\a\-\d\e\v\-\3\1* ]] + [23-01-15 09:20:02] [[ rdma-qe-30.rdma.lab.eng.rdu2.redhat.com == *\r\d\m\a\-\d\e\v\-\3\0* ]] + [23-01-15 09:20:02] [[ rdma-qe-30.rdma.lab.eng.rdu2.redhat.com == *\r\d\m\a\-\d\e\v\-\3\1* ]] + [23-01-15 09:20:02] '[' -z rdma-qe-30.rdma.lab.eng.rdu2.redhat.com -a -z rdma-qe-31.rdma.lab.eng.rdu2.redhat.com ']' + [23-01-15 09:20:02] RQA_set_servers_clients ++ [23-01-15 09:20:02] echo rdma-qe-31.rdma.lab.eng.rdu2.redhat.com ++ [23-01-15 09:20:02] awk -F . '{ print $1}' + [23-01-15 09:20:02] s_server_name=rdma-qe-31 ++ [23-01-15 09:20:02] echo rdma-qe-30.rdma.lab.eng.rdu2.redhat.com ++ [23-01-15 09:20:02] awk -F . '{ print $1}' + [23-01-15 09:20:02] s_client_name=rdma-qe-30 + [23-01-15 09:20:02] grep 'export SERVERS=' /root/.bashrc + [23-01-15 09:20:02] echo 'export SERVERS="rdma-qe-31"' + [23-01-15 09:20:02] grep 'export CLIENTS=' /root/.bashrc + [23-01-15 09:20:02] echo 'export CLIENTS="rdma-qe-30"' + [23-01-15 09:20:02] source /root/.bashrc ++ [23-01-15 09:20:02] '[' -f /etc/bashrc ']' ++ [23-01-15 09:20:02] . /etc/bashrc +++ [23-01-15 09:20:02] '[' -z Y ']' ++ [23-01-15 09:20:02] '[' vt100 = vt102 ']' ++ [23-01-15 09:20:02] alias 'grep=/bin/grep --color=tty ' ++ [23-01-15 09:20:02] alias vi=vim ++ [23-01-15 09:20:02] export EDITOR=vim ++ [23-01-15 09:20:02] EDITOR=vim ++ [23-01-15 09:20:02] RDMA_LOOKASIDE=http://download-node-02.eng.bos.redhat.com/qa/rhts/lookaside/rdma-testing +++ [23-01-15 09:20:02] which rhts-reboot +++ [23-01-15 09:20:02] alias +++ [23-01-15 09:20:02] /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot rhts-reboot +++ [23-01-15 09:20:02] eval declare -f ++++ [23-01-15 09:20:02] declare -f ++ [23-01-15 09:20:02] '[' '!' -x /usr/bin/rhts-reboot ']' ++ [23-01-15 09:20:02] alias shutdown=rhts-reboot ++ [23-01-15 09:20:02] export SERVERS=rdma-qe-31 ++ [23-01-15 09:20:02] SERVERS=rdma-qe-31 ++ [23-01-15 09:20:02] export CLIENTS=rdma-qe-30 ++ [23-01-15 09:20:02] CLIENTS=rdma-qe-30 + [23-01-15 09:20:02] RQA_rhts_or_dev_mode multi env + [23-01-15 09:20:02] export RUN_NUMBER=0 + [23-01-15 09:20:02] RUN_NUMBER=0 ++ [23-01-15 09:20:02] grep -i 'Job whiteboard:' /etc/motd ++ [23-01-15 09:20:02] awk -F '[' '{print $2}' ++ [23-01-15 09:20:02] sed 's/\]//g' + [23-01-15 09:20:02] merge_request= + [23-01-15 09:20:02] [[ ! -z '' ]] + [23-01-15 09:20:02] test -z 7432005 + [23-01-15 09:20:02] echo 'Variable JOBID set, we'\''re running on RHTS' Variable JOBID set, we're running on RHTS +++ [23-01-15 09:20:02] pwd ++ [23-01-15 09:20:02] basename /mnt/tests/gitlab.cee.redhat.com/kernel-qe/kernel/-/archive/master/kernel-master.zip/infiniband/env_setup + [23-01-15 09:20:02] '[' env_setup == nfsordma_sh ']' + [23-01-15 09:20:02] exec + [23-01-15 09:20:02] exec + [23-01-15 09:20:02] '[' multi == multi ']' + [23-01-15 09:20:02] echo 'Clients: rdma-qe-30' Clients: rdma-qe-30 + [23-01-15 09:20:02] echo 'Servers: rdma-qe-31' Servers: rdma-qe-31 + [23-01-15 09:20:02] RQA_system_info_for_debug + [23-01-15 09:20:02] grep -i distro /etc/motd + [23-01-15 09:20:02] tr -d ' ' DISTRO=CentOS-Stream-9-20230109.0 + [23-01-15 09:20:02] cat /etc/redhat-release CentOS Stream release 9 + [23-01-15 09:20:02] uname -a Linux rdma-qe-31.rdma.lab.eng.rdu2.redhat.com 5.14.0-235.rt14.236.1639_747671795.el9.x86_64 #1 SMP PREEMPT_RT Sun Jan 15 12:47:38 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux + [23-01-15 09:20:02] cat /proc/cmdline BOOT_IMAGE=(hd0,gpt2)/vmlinuz-5.14.0-235.rt14.236.1639_747671795.el9.x86_64 root=UUID=e6dfca68-68e9-4ada-872e-5d60193d151c ro efi=runtime resume=UUID=b1f3698a-6dd5-46a4-827b-26a8975186b6 console=ttyS0,115200n81 crashkernel=1G-4G:384M,4G-16G:512M,16G-64G:1G,64G-128G:2G,128G-:4G + [23-01-15 09:20:02] rpm -q rdma-core linux-firmware rdma-core-41.0-3.el9.x86_64 linux-firmware-20221214-129.el9.noarch + [23-01-15 09:20:02] tail /sys/class/infiniband/bnxt_re0/fw_ver /sys/class/infiniband/bnxt_re1/fw_ver ==> /sys/class/infiniband/bnxt_re0/fw_ver <== 214.0.182.0 ==> /sys/class/infiniband/bnxt_re1/fw_ver <== 214.0.182.0 + [23-01-15 09:20:02] lspci + [23-01-15 09:20:02] grep -i -e ethernet -e infiniband -e omni -e ConnectX 44:00.0 Ethernet controller: Broadcom Inc. and subsidiaries BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller 44:00.1 Ethernet controller: Broadcom Inc. and subsidiaries BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller c5:00.0 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01) c5:00.1 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01) c5:00.2 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01) c5:00.3 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01) + [23-01-15 09:20:02] lscpu Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Address sizes: 48 bits physical, 48 bits virtual Byte Order: Little Endian CPU(s): 48 On-line CPU(s) list: 0-47 Vendor ID: AuthenticAMD BIOS Vendor ID: Advanced Micro Devices, Inc. Model name: AMD EPYC 7402P 24-Core Processor BIOS Model name: AMD EPYC 7402P 24-Core Processor CPU family: 23 Model: 49 Thread(s) per core: 2 Core(s) per socket: 24 Socket(s): 1 Stepping: 0 Frequency boost: enabled CPU max MHz: 2800.0000 CPU min MHz: 1500.0000 BogoMIPS: 5589.37 Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca Virtualization: AMD-V L1d cache: 768 KiB (24 instances) L1i cache: 768 KiB (24 instances) L2 cache: 12 MiB (24 instances) L3 cache: 128 MiB (8 instances) NUMA node(s): 1 NUMA node0 CPU(s): 0-47 Vulnerability Itlb multihit: Not affected Vulnerability L1tf: Not affected Vulnerability Mds: Not affected Vulnerability Meltdown: Not affected Vulnerability Mmio stale data: Not affected Vulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization Vulnerability Spectre v2: Mitigation; Retpolines, IBPB conditional, STIBP always-on, RSB filling, PBRSB-eIBRS Not affected Vulnerability Srbds: Not affected Vulnerability Tsx async abort: Not affected + [23-01-15 09:20:02] ibstat CA 'bnxt_re0' CA type: Broadcom NetXtreme-C/E RoCE Driver HCA Number of ports: 1 Firmware version: 214.0.182.0 Hardware version: 0x14e4 Node GUID: 0x020af7fffe97b162 System image GUID: 0x020af7fffe97b162 Port 1: State: Active Physical state: LinkUp Rate: 25 Base lid: 0 LMC: 0 SM lid: 0 Capability mask: 0x001d0000 Port GUID: 0x020af7fffe97b162 Link layer: Ethernet CA 'bnxt_re1' CA type: Broadcom NetXtreme-C/E RoCE Driver HCA Number of ports: 1 Firmware version: 214.0.182.0 Hardware version: 0x14e4 Node GUID: 0x020af7fffe97b163 System image GUID: 0x020af7fffe97b163 Port 1: State: Down Physical state: Disabled Rate: 2.5 Base lid: 0 LMC: 0 SM lid: 0 Capability mask: 0x001d0000 Port GUID: 0x020af7fffe97b163 Link layer: Ethernet + [23-01-15 09:20:05] ibstatus Infiniband device 'bnxt_re0' port 1 status: default gid: fe80:0000:0000:0000:020a:f7ff:fe97:b162 base lid: 0x0 sm lid: 0x0 state: 4: ACTIVE phys state: 5: LinkUp rate: 25 Gb/sec (1X EDR) link_layer: Ethernet Infiniband device 'bnxt_re1' port 1 status: default gid: fe80:0000:0000:0000:020a:f7ff:fe97:b163 base lid: 0x0 sm lid: 0x0 state: 1: DOWN phys state: 3: Disabled rate: 2.5 Gb/sec (1X SDR) link_layer: Ethernet + [23-01-15 09:20:05] /usr/bin/ibv_devinfo -v hca_id: bnxt_re0 transport: InfiniBand (0) fw_ver: 214.0.182.0 node_guid: 020a:f7ff:fe97:b162 sys_image_guid: 020a:f7ff:fe97:b162 vendor_id: 0x14e4 vendor_part_id: 5847 hw_ver: 0x1402 phys_port_cnt: 1 max_mr_size: 0x8000000000 page_size_cap: 0x201000 max_qp: 65408 max_qp_wr: 65406 device_cap_flags: 0x01225d81 RESIZE_MAX_WR CURR_QP_STATE_MOD SHUTDOWN_PORT PORT_ACTIVE_EVENT SYS_IMAGE_GUID RC_RNR_NAK_GEN N_NOTIFY_CQ MEM_WINDOW MEM_MGT_EXTENSIONS MEM_WINDOW_TYPE_2B max_sge: 6 max_sge_rd: 6 max_cq: 65535 max_cqe: 1048575 max_mr: 262143 max_pd: 65536 max_qp_rd_atom: 16 max_ee_rd_atom: 0 max_res_rd_atom: 0 max_qp_init_rd_atom: 126 max_ee_init_rd_atom: 0 atomic_cap: ATOMIC_NONE (0) max_ee: 0 max_rdd: 0 max_mw: 524288 max_raw_ipv6_qp: 0 max_raw_ethy_qp: 524288 max_mcast_grp: 0 max_mcast_qp_attach: 0 max_total_mcast_qp_attach: 0 max_ah: 524288 max_fmr: 0 max_srq: 32767 max_srq_wr: 1048574 max_srq_sge: 6 max_pkeys: 1 local_ca_ack_delay: 16 general_odp_caps: rc_odp_caps: NO SUPPORT uc_odp_caps: NO SUPPORT ud_odp_caps: NO SUPPORT xrc_odp_caps: NO SUPPORT completion_timestamp_mask not supported core clock not supported device_cap_flags_ex: 0x1225D81 tso_caps: max_tso: 0 rss_caps: max_rwq_indirection_tables: 0 max_rwq_indirection_table_size: 0 rx_hash_function: 0x0 rx_hash_fields_mask: 0x0 max_wq_type_rq: 0 packet_pacing_caps: qp_rate_limit_min: 0kbps qp_rate_limit_max: 0kbps tag matching not supported num_comp_vectors: 8 port: 1 state: PORT_ACTIVE (4) max_mtu: 4096 (5) active_mtu: 4096 (5) sm_lid: 0 port_lid: 0 port_lmc: 0x00 link_layer: Ethernet max_msg_sz: 0x40000000 port_cap_flags: 0x041d0000 port_cap_flags2: 0x0000 max_vl_num: 8 (4) bad_pkey_cntr: 0x0 qkey_viol_cntr: 0x0 sm_sl: 0 pkey_tbl_len: 65535 gid_tbl_len: 256 subnet_timeout: 0 init_type_reply: 0 active_width: 1X (1) active_speed: 25.0 Gbps (32) phys_state: LINK_UP (5) GID[ 0]: fe80:0000:0000:0000:020a:f7ff:fe97:b162, RoCE v1 GID[ 1]: fe80::20a:f7ff:fe97:b162, RoCE v2 GID[ 2]: 0000:0000:0000:0000:0000:ffff:ac1f:281f, RoCE v1 GID[ 3]: ::ffff:172.31.40.31, RoCE v2 GID[ 4]: 0000:0000:0000:0000:0000:ffff:ac1f:2b1f, RoCE v1 GID[ 5]: ::ffff:172.31.43.31, RoCE v2 GID[ 6]: fe80:0000:0000:0000:020a:f7ff:fe97:b162, RoCE v1 GID[ 7]: fe80::20a:f7ff:fe97:b162, RoCE v2 GID[ 8]: 0000:0000:0000:0000:0000:ffff:ac1f:2d1f, RoCE v1 GID[ 9]: ::ffff:172.31.45.31, RoCE v2 GID[ 10]: fe80:0000:0000:0000:020a:f7ff:fe97:b162, RoCE v1 GID[ 11]: fe80::20a:f7ff:fe97:b162, RoCE v2 hca_id: bnxt_re1 transport: InfiniBand (0) fw_ver: 214.0.182.0 node_guid: 020a:f7ff:fe97:b163 sys_image_guid: 020a:f7ff:fe97:b163 vendor_id: 0x14e4 vendor_part_id: 5847 hw_ver: 0x1402 phys_port_cnt: 1 max_mr_size: 0x8000000000 page_size_cap: 0x201000 max_qp: 65408 max_qp_wr: 65406 device_cap_flags: 0x01225d81 RESIZE_MAX_WR CURR_QP_STATE_MOD SHUTDOWN_PORT PORT_ACTIVE_EVENT SYS_IMAGE_GUID RC_RNR_NAK_GEN N_NOTIFY_CQ MEM_WINDOW MEM_MGT_EXTENSIONS MEM_WINDOW_TYPE_2B max_sge: 6 max_sge_rd: 6 max_cq: 65535 max_cqe: 1048575 max_mr: 262143 max_pd: 65536 max_qp_rd_atom: 16 max_ee_rd_atom: 0 max_res_rd_atom: 0 max_qp_init_rd_atom: 126 max_ee_init_rd_atom: 0 atomic_cap: ATOMIC_NONE (0) max_ee: 0 max_rdd: 0 max_mw: 524288 max_raw_ipv6_qp: 0 max_raw_ethy_qp: 524288 max_mcast_grp: 0 max_mcast_qp_attach: 0 max_total_mcast_qp_attach: 0 max_ah: 524288 max_fmr: 0 max_srq: 32767 max_srq_wr: 1048574 max_srq_sge: 6 max_pkeys: 1 local_ca_ack_delay: 16 general_odp_caps: rc_odp_caps: NO SUPPORT uc_odp_caps: NO SUPPORT ud_odp_caps: NO SUPPORT xrc_odp_caps: NO SUPPORT completion_timestamp_mask not supported core clock not supported device_cap_flags_ex: 0x1225D81 tso_caps: max_tso: 0 rss_caps: max_rwq_indirection_tables: 0 max_rwq_indirection_table_size: 0 rx_hash_function: 0x0 rx_hash_fields_mask: 0x0 max_wq_type_rq: 0 packet_pacing_caps: qp_rate_limit_min: 0kbps qp_rate_limit_max: 0kbps tag matching not supported num_comp_vectors: 8 port: 1 state: PORT_DOWN (1) max_mtu: 4096 (5) active_mtu: 1024 (3) sm_lid: 0 port_lid: 0 port_lmc: 0x00 link_layer: Ethernet max_msg_sz: 0x40000000 port_cap_flags: 0x041d0000 port_cap_flags2: 0x0000 max_vl_num: 8 (4) bad_pkey_cntr: 0x0 qkey_viol_cntr: 0x0 sm_sl: 0 pkey_tbl_len: 65535 gid_tbl_len: 256 subnet_timeout: 0 init_type_reply: 0 active_width: 1X (1) active_speed: 2.5 Gbps (1) phys_state: DISABLED (3) GID[ 0]: fe80:0000:0000:0000:020a:f7ff:fe97:b163, RoCE v1 GID[ 1]: fe80::20a:f7ff:fe97:b163, RoCE v2 + [23-01-15 09:20:05] iblinkinfo ibwarn: [15176] get_abi_version: can't read ABI version from /sys/class/infiniband_mad/abi_version (No such file or directory): is ib_umad module loaded? ibwarn: [15176] mad_rpc_open_port: can't open UMAD port ((null):0) Failed to open (null) port 0 + [23-01-15 09:20:05] [[ '' == *\o\p\a* ]] + [23-01-15 09:20:05] ip addr show 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: bnxt_roce: mtu 9000 qdisc mq state UP group default qlen 1000 link/ether 00:0a:f7:97:b1:62 brd ff:ff:ff:ff:ff:ff altname enp68s0f0np0 altname ens2f0np0 inet 172.31.40.31/24 brd 172.31.40.255 scope global dynamic noprefixroute bnxt_roce valid_lft 3379sec preferred_lft 3379sec inet6 fe80::20a:f7ff:fe97:b162/64 scope link noprefixroute valid_lft forever preferred_lft forever 3: bnxt_off: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 00:0a:f7:97:b1:63 brd ff:ff:ff:ff:ff:ff altname enp68s0f1np1 altname ens2f1np1 4: lom_1: mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 48:df:37:ad:f6:f0 brd ff:ff:ff:ff:ff:ff altname enp197s0f0 altname eno5 inet 10.1.217.81/24 brd 10.1.217.255 scope global dynamic noprefixroute lom_1 valid_lft 86181sec preferred_lft 86181sec inet6 2620:52:0:1d9:4adf:37ff:fead:f6f0/64 scope global dynamic noprefixroute valid_lft 2591946sec preferred_lft 604746sec inet6 fe80::4adf:37ff:fead:f6f0/64 scope link noprefixroute valid_lft forever preferred_lft forever 5: lom_2: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 48:df:37:ad:f6:f1 brd ff:ff:ff:ff:ff:ff altname enp197s0f1 altname eno6 6: lom_3: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 48:df:37:ad:f6:f2 brd ff:ff:ff:ff:ff:ff altname enp197s0f2 altname eno7 7: lom_4: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 48:df:37:ad:f6:f3 brd ff:ff:ff:ff:ff:ff altname enp197s0f3 altname eno8 8: bnxt_roce.43@bnxt_roce: mtu 9000 qdisc noqueue state UP group default qlen 1000 link/ether 00:0a:f7:97:b1:62 brd ff:ff:ff:ff:ff:ff inet 172.31.43.31/24 brd 172.31.43.255 scope global dynamic noprefixroute bnxt_roce.43 valid_lft 3379sec preferred_lft 3379sec inet6 fe80::20a:f7ff:fe97:b162/64 scope link noprefixroute valid_lft forever preferred_lft forever 9: bnxt_roce.45@bnxt_roce: mtu 9000 qdisc noqueue state UP group default qlen 1000 link/ether 00:0a:f7:97:b1:62 brd ff:ff:ff:ff:ff:ff inet 172.31.45.31/24 brd 172.31.45.255 scope global dynamic noprefixroute bnxt_roce.45 valid_lft 3379sec preferred_lft 3379sec inet6 fe80::20a:f7ff:fe97:b162/64 scope link noprefixroute valid_lft forever preferred_lft forever + [23-01-15 09:20:05] PY_FILE=/root/py_parse_out.txt + [23-01-15 09:20:05] rm -f /root/py_parse_out.txt + [23-01-15 09:20:05] export result=PASS + [23-01-15 09:20:05] result=PASS + [23-01-15 09:20:05] export TEST=/kernel/infiniband/env_setup/standalone + [23-01-15 09:20:05] TEST=/kernel/infiniband/env_setup/standalone + [23-01-15 09:20:05] ip address show + [23-01-15 09:20:05] grep 'inet 172.31.' + [23-01-15 09:20:05] [[ 0 -ne 0 ]] + [23-01-15 09:20:05] install_ca_trust + [23-01-15 09:20:05] local cert_path=/etc/pki/ca-trust/source/anchors + [23-01-15 09:20:05] '[' -f /etc/pki/ca-trust/source/anchors/legacy.crt -a -f /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt ']' + [23-01-15 09:20:05] wget --no-check-certificate -P /etc/pki/ca-trust/source/anchors https://password.corp.redhat.com/legacy.crt --2023-01-15 09:20:05-- https://password.corp.redhat.com/legacy.crt Resolving password.corp.redhat.com (password.corp.redhat.com)... 10.2.111.154 Connecting to password.corp.redhat.com (password.corp.redhat.com)|10.2.111.154|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 1338 (1.3K) Saving to: ‘/etc/pki/ca-trust/source/anchors/legacy.crt’ 0K . 100% 47.2M=0s 2023-01-15 09:20:05 (47.2 MB/s) - ‘/etc/pki/ca-trust/source/anchors/legacy.crt’ saved [1338/1338] + [23-01-15 09:20:05] wget --no-check-certificate -P /etc/pki/ca-trust/source/anchors https://password.corp.redhat.com/RH-IT-Root-CA.crt --2023-01-15 09:20:05-- https://password.corp.redhat.com/RH-IT-Root-CA.crt Resolving password.corp.redhat.com (password.corp.redhat.com)... 10.2.111.154 Connecting to password.corp.redhat.com (password.corp.redhat.com)|10.2.111.154|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 1517 (1.5K) Saving to: ‘/etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt.1’ 0K . 100% 76.3M=0s 2023-01-15 09:20:06 (76.3 MB/s) - ‘/etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt.1’ saved [1517/1517] + [23-01-15 09:20:06] update-ca-trust enable + [23-01-15 09:20:07] update-ca-trust extract + [23-01-15 09:20:08] RQA_rhel8_brew_repo ++ [23-01-15 09:20:08] RQA_get_rhel_major ++ [23-01-15 09:20:08] awk -F . '{print $1}' +++ [23-01-15 09:20:08] RQA_get_rhel_release +++ [23-01-15 09:20:08] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release ++ [23-01-15 09:20:08] echo 9 + [23-01-15 09:20:08] local major=9 + [23-01-15 09:20:08] (( major == 8 )) + [23-01-15 09:20:08] pek2_lab_setup ++ [23-01-15 09:20:08] RQA_get_rdma_hostname ++ [23-01-15 09:20:08] hostname -A ++ [23-01-15 09:20:08] grep -o 'rdma.*' ++ [23-01-15 09:20:08] awk '{print $1}' ++ [23-01-15 09:20:08] awk -F . '{print $1}' + [23-01-15 09:20:08] [[ rdma-qe-31 == *\r\d\m\a\0* ]] + [23-01-15 09:20:08] common_setup ++ [23-01-15 09:20:08] RQA_get_rdma_hostname ++ [23-01-15 09:20:08] hostname -A ++ [23-01-15 09:20:08] grep -o 'rdma.*' ++ [23-01-15 09:20:08] awk '{print $1}' ++ [23-01-15 09:20:08] awk -F . '{print $1}' + [23-01-15 09:20:08] short_hostname=rdma-qe-31 + [23-01-15 09:20:08] RDMA_DRIVER= + [23-01-15 09:20:08] RDMA_NETWORK= + [23-01-15 09:20:08] '[' '!' -z ']' + [23-01-15 09:20:08] '[' '!' -z roce ']' + [23-01-15 09:20:08] RDMA_NETWORK=roce + [23-01-15 09:20:08] inv_file=json/rdma-qe-31.json + [23-01-15 09:20:08] '[' '!' -f json/rdma-qe-31.json ']' + [23-01-15 09:20:08] set -o pipefail + [23-01-15 09:20:08] python host_data_parser.py json/rdma-qe-31.json '' roce + [23-01-15 09:20:08] tee /root/py_parse_out.txt export RDMA_DRIVER="bnxt" export RDMA_NETWORK="roce" export HCA_ID="bnxt_re0" export HCA_RATE="25 Gb/sec" export HCA_ABRV="BCM57414" export DEVICE_ID="bnxt_roce" export DEVICE_MAC="00:0a:f7:97:b1:62" export DEVICE_PORT="1" + [23-01-15 09:20:08] '[' 0 -ne 0 ']' + [23-01-15 09:20:08] source /root/py_parse_out.txt ++ [23-01-15 09:20:08] export RDMA_DRIVER=bnxt ++ [23-01-15 09:20:08] RDMA_DRIVER=bnxt ++ [23-01-15 09:20:08] export RDMA_NETWORK=roce ++ [23-01-15 09:20:08] RDMA_NETWORK=roce ++ [23-01-15 09:20:08] export HCA_ID=bnxt_re0 ++ [23-01-15 09:20:08] HCA_ID=bnxt_re0 ++ [23-01-15 09:20:08] export 'HCA_RATE=25 Gb/sec' ++ [23-01-15 09:20:08] HCA_RATE='25 Gb/sec' ++ [23-01-15 09:20:08] export HCA_ABRV=BCM57414 ++ [23-01-15 09:20:08] HCA_ABRV=BCM57414 ++ [23-01-15 09:20:08] export DEVICE_ID=bnxt_roce ++ [23-01-15 09:20:08] DEVICE_ID=bnxt_roce ++ [23-01-15 09:20:08] export DEVICE_MAC=00:0a:f7:97:b1:62 ++ [23-01-15 09:20:08] DEVICE_MAC=00:0a:f7:97:b1:62 ++ [23-01-15 09:20:08] export DEVICE_PORT=1 ++ [23-01-15 09:20:08] DEVICE_PORT=1 + [23-01-15 09:20:08] RQA_check_result -r 0 -t 'Source python parser output' + [23-01-15 09:20:08] local test_pass=0 + [23-01-15 09:20:08] local test_skip=777 + [23-01-15 09:20:08] test 4 -gt 0 + [23-01-15 09:20:08] case $1 in + [23-01-15 09:20:08] local rc=0 + [23-01-15 09:20:08] shift + [23-01-15 09:20:08] shift + [23-01-15 09:20:08] test 2 -gt 0 + [23-01-15 09:20:08] case $1 in + [23-01-15 09:20:08] local 'msg=Source python parser output' + [23-01-15 09:20:08] shift + [23-01-15 09:20:08] shift + [23-01-15 09:20:08] test 0 -gt 0 + [23-01-15 09:20:08] '[' -z 0 -o -z 'Source python parser output' ']' + [23-01-15 09:20:08] '[' -z '' ']' + [23-01-15 09:20:08] source /etc/environment + [23-01-15 09:20:08] '[' -z '' ']' + [23-01-15 09:20:08] __RQA_create_test_result_file ++ [23-01-15 09:20:08] rpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}' rdma-core + [23-01-15 09:20:08] local rdma_version=rdma-core-41.0-3.el9 ++ [23-01-15 09:20:08] echo /kernel/infiniband/env_setup/standalone ++ [23-01-15 09:20:08] sed -r 's/\/kernel\/infiniband\///;s/\/server//;s/\/client//;s/\/standalone//' + [23-01-15 09:20:08] TEST_NAME=env_setup + [23-01-15 09:20:08] [[ ! -z rdma-qe-31 ]] + [23-01-15 09:20:08] [[ ! -z rdma-qe-30 ]] + [23-01-15 09:20:08] hosts_list=rdma-qe-31/rdma-qe-30 + [23-01-15 09:20:08] local _bkrjb= ++ [23-01-15 09:20:08] grep JOBID= /etc/motd ++ [23-01-15 09:20:08] tr -d ' ' ++ [23-01-15 09:20:08] sed s/JOBID=/J:/g ++ [23-01-15 09:20:08] head -1 + [23-01-15 09:20:08] _bkrjb=J:7432005 + [23-01-15 09:20:08] [[ ! -z J:7432005 ]] + [23-01-15 09:20:08] _bkrjb=' & Beaker job J:7432005' ++ [23-01-15 09:20:08] echo results_env_setup.txt ++ [23-01-15 09:20:08] sed 's/\//-/g' + [23-01-15 09:20:08] local result_filename=results_env_setup.txt ++ [23-01-15 09:20:08] mktemp -d + [23-01-15 09:20:08] TEST_RESULT_FILE=/tmp/tmp.mXlwpDN8bq/results_env_setup.txt + [23-01-15 09:20:08] echo 'env_setup test results on rdma-qe-31/rdma-qe-30 & Beaker job J:7432005:' ++ [23-01-15 09:20:08] uname -r + [23-01-15 09:20:08] echo '5.14.0-235.rt14.236.1639_747671795.el9.x86_64, rdma-core-41.0-3.el9, bnxt, roce, BCM57414 & bnxt_re0' + [23-01-15 09:20:08] echo ' Result | Status | Test' + [23-01-15 09:20:08] echo ' ---------+--------+------------------------------------' + [23-01-15 09:20:08] sed -i /TEST_RESULT_FILE/d /etc/environment + [23-01-15 09:20:08] echo TEST_RESULT_FILE=/tmp/tmp.mXlwpDN8bq/results_env_setup.txt + [23-01-15 09:20:08] source /etc/environment ++ [23-01-15 09:20:08] TEST_RESULT_FILE=/tmp/tmp.mXlwpDN8bq/results_env_setup.txt + [23-01-15 09:20:08] '[' 0 -eq 0 ']' + [23-01-15 09:20:08] local test_result=PASS + [23-01-15 09:20:08] printf '%10s | %6s | %s\n' PASS 0 'Source python parser output' + [23-01-15 09:20:08] set +x --- - TEST RESULT FOR env_setup - Test: Source python parser output - Result: PASS - Return: 0 --- + [23-01-15 09:20:08] [[ bnxt == \r\x\e ]] + [23-01-15 09:20:08] RQA_bring_up_network roce bnxt + [23-01-15 09:20:08] '[' 2 -ne 2 ']' + [23-01-15 09:20:08] local _RDMA_NETWORK=roce + [23-01-15 09:20:08] local _RDMA_DRIVER=bnxt ++ [23-01-15 09:20:08] RQA_get_rdma_hostname ++ [23-01-15 09:20:08] hostname -A ++ [23-01-15 09:20:08] grep -o 'rdma.*' ++ [23-01-15 09:20:08] awk '{print $1}' ++ [23-01-15 09:20:08] awk -F . '{print $1}' + [23-01-15 09:20:08] local rdma_hostname=rdma-qe-31 + [23-01-15 09:20:09] [[ rdma-qe-31 == *\r\d\m\a\0* ]] + [23-01-15 09:20:09] [[ ! -z rdma-qe-30 ]] + [23-01-15 09:20:09] [[ ! -z rdma-qe-31 ]] ++ [23-01-15 09:20:09] RQA_hn_tail rdma-qe-31 ++ [23-01-15 09:20:09] echo rdma-qe-31 ++ [23-01-15 09:20:09] sed -e 's/\..*//' -e s/rdma-// + [23-01-15 09:20:09] local alias_server=roce-qe-31 ++ [23-01-15 09:20:09] cat /etc/hosts ++ [23-01-15 09:20:09] grep roce-qe-31 ++ [23-01-15 09:20:09] awk -F ' ' '{print $1}' + [23-01-15 09:20:09] IP_SERVER=172.31.40.31 ++ [23-01-15 09:20:09] RQA_hn_tail rdma-qe-30 ++ [23-01-15 09:20:09] echo rdma-qe-30 ++ [23-01-15 09:20:09] sed -e 's/\..*//' -e s/rdma-// + [23-01-15 09:20:09] local alias_client=roce-qe-30 ++ [23-01-15 09:20:09] cat /etc/hosts ++ [23-01-15 09:20:09] grep roce-qe-30 ++ [23-01-15 09:20:09] awk -F ' ' '{print $1}' + [23-01-15 09:20:09] IP_CLIENT=172.31.40.30 + [23-01-15 09:20:09] local bad=0 + [23-01-15 09:20:09] ping -c3 172.31.40.31 PING 172.31.40.31 (172.31.40.31) 56(84) bytes of data. 64 bytes from 172.31.40.31: icmp_seq=1 ttl=64 time=0.035 ms 64 bytes from 172.31.40.31: icmp_seq=2 ttl=64 time=0.048 ms 64 bytes from 172.31.40.31: icmp_seq=3 ttl=64 time=0.026 ms --- 172.31.40.31 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2075ms rtt min/avg/max/mdev = 0.026/0.036/0.048/0.009 ms + [23-01-15 09:20:11] (( bad+=0 )) + [23-01-15 09:20:11] ping -c3 172.31.40.30 PING 172.31.40.30 (172.31.40.30) 56(84) bytes of data. From 172.31.40.31 icmp_seq=1 Destination Host Unreachable From 172.31.40.31 icmp_seq=2 Destination Host Unreachable From 172.31.40.31 icmp_seq=3 Destination Host Unreachable --- 172.31.40.30 ping statistics --- 3 packets transmitted, 0 received, +3 errors, 100% packet loss, time 2040ms pipe 3 + [23-01-15 09:20:14] (( bad+=1 )) + [23-01-15 09:20:14] '[' 1 -eq 0 ']' ++ [23-01-15 09:20:14] RQA_get_driver_modules ++ [23-01-15 09:20:14] case ${RDMA_DRIVER} in ++ [23-01-15 09:20:14] echo 'bnxt_re bnxt_en' + [23-01-15 09:20:14] modprobe bnxt_re bnxt_en + [23-01-15 09:20:14] [[ roce == *\o\p\a* ]] + [23-01-15 09:20:14] RQA_pkg_install opensm + [23-01-15 09:20:14] PKG_LIST= + [23-01-15 09:20:14] for p in "$@" + [23-01-15 09:20:14] rpm -q opensm package opensm is not installed + [23-01-15 09:20:14] PKG_LIST=' opensm' + [23-01-15 09:20:14] '[' '!' -z ' opensm' ']' + [23-01-15 09:20:14] dnf install -y --setopt=strict=0 --nogpgcheck opensm Last metadata expiration check: 0:10:38 ago on Sun 15 Jan 2023 09:09:37 AM EST. Dependencies resolved. ================================================================================ Package Architecture Version Repository Size ================================================================================ Installing: opensm x86_64 3.3.24-2.el9 BUILDROOT-C9S 504 k Transaction Summary ================================================================================ Install 1 Package Total download size: 504 k Installed size: 1.2 M Downloading Packages: opensm-3.3.24-2.el9.x86_64.rpm 11 MB/s | 504 kB 00:00 -------------------------------------------------------------------------------- Total 10 MB/s | 504 kB 00:00 Running transaction check Transaction check succeeded. Running transaction test Transaction test succeeded. Running transaction Preparing : 1/1 Installing : opensm-3.3.24-2.el9.x86_64 1/1 Running scriptlet: opensm-3.3.24-2.el9.x86_64 1/1 Verifying : opensm-3.3.24-2.el9.x86_64 1/1 Installed: opensm-3.3.24-2.el9.x86_64 Complete! + [23-01-15 09:20:19] RQA_sys_service enable opensm + [23-01-15 09:20:19] SERVICE_RETURN=1 + [23-01-15 09:20:19] '[' 2 -ne 2 ']' + [23-01-15 09:20:19] test 2 -gt 0 + [23-01-15 09:20:19] case $1 in + [23-01-15 09:20:19] action=enable + [23-01-15 09:20:19] shift + [23-01-15 09:20:19] test 1 -gt 0 + [23-01-15 09:20:19] case $1 in + [23-01-15 09:20:19] serv=opensm + [23-01-15 09:20:19] shift + [23-01-15 09:20:19] test 0 -gt 0 ++ [23-01-15 09:20:19] RQA_get_rhel_major ++ [23-01-15 09:20:19] awk -F . '{print $1}' +++ [23-01-15 09:20:19] RQA_get_rhel_release +++ [23-01-15 09:20:19] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release ++ [23-01-15 09:20:19] echo 9 + [23-01-15 09:20:19] '[' 9 -ge 7 ']' + [23-01-15 09:20:19] /usr/bin/systemctl enable opensm Created symlink /etc/systemd/system/network.target.wants/opensm.service → /usr/lib/systemd/system/opensm.service. + [23-01-15 09:20:19] SERVICE_RETURN=0 + [23-01-15 09:20:19] return 0 + [23-01-15 09:20:19] RQA_sys_service restart opensm + [23-01-15 09:20:19] SERVICE_RETURN=1 + [23-01-15 09:20:19] '[' 2 -ne 2 ']' + [23-01-15 09:20:19] test 2 -gt 0 + [23-01-15 09:20:19] case $1 in + [23-01-15 09:20:19] action=restart + [23-01-15 09:20:19] shift + [23-01-15 09:20:19] test 1 -gt 0 + [23-01-15 09:20:19] case $1 in + [23-01-15 09:20:19] serv=opensm + [23-01-15 09:20:19] shift + [23-01-15 09:20:19] test 0 -gt 0 ++ [23-01-15 09:20:19] RQA_get_rhel_major ++ [23-01-15 09:20:19] awk -F . '{print $1}' +++ [23-01-15 09:20:19] RQA_get_rhel_release +++ [23-01-15 09:20:19] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release ++ [23-01-15 09:20:19] echo 9 + [23-01-15 09:20:19] '[' 9 -ge 7 ']' + [23-01-15 09:20:19] /usr/bin/systemctl restart opensm + [23-01-15 09:20:19] SERVICE_RETURN=0 + [23-01-15 09:20:19] return 0 + [23-01-15 09:20:19] sleep 3 + [23-01-15 09:20:22] modprobe -r ib_ipoib + [23-01-15 09:20:22] sleep 3 + [23-01-15 09:20:25] modprobe ib_ipoib + [23-01-15 09:20:25] sleep 3 + [23-01-15 09:20:28] RQA_sys_service status opensm + [23-01-15 09:20:28] SERVICE_RETURN=1 + [23-01-15 09:20:28] '[' 2 -ne 2 ']' + [23-01-15 09:20:28] test 2 -gt 0 + [23-01-15 09:20:28] case $1 in + [23-01-15 09:20:28] action=status + [23-01-15 09:20:28] shift + [23-01-15 09:20:28] test 1 -gt 0 + [23-01-15 09:20:28] case $1 in + [23-01-15 09:20:28] serv=opensm + [23-01-15 09:20:28] shift + [23-01-15 09:20:28] test 0 -gt 0 ++ [23-01-15 09:20:28] RQA_get_rhel_major ++ [23-01-15 09:20:28] awk -F . '{print $1}' +++ [23-01-15 09:20:28] RQA_get_rhel_release +++ [23-01-15 09:20:28] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release ++ [23-01-15 09:20:28] echo 9 + [23-01-15 09:20:28] '[' 9 -ge 7 ']' + [23-01-15 09:20:28] /usr/bin/systemctl status opensm ● opensm.service - Starts the OpenSM InfiniBand fabric Subnet Manager Loaded: loaded (/usr/lib/systemd/system/opensm.service; enabled; preset: disabled) Active: active (running) since Sun 2023-01-15 09:20:19 EST; 9s ago Docs: man:opensm Process: 15671 ExecStart=/usr/libexec/opensm-launch (code=exited, status=0/SUCCESS) Main PID: 15672 (opensm-launch) Tasks: 2 (limit: 252286) Memory: 2.2M CPU: 6.987s CGroup: /system.slice/opensm.service ├─15672 /usr/bin/bash /usr/libexec/opensm-launch └─15744 sleep 30 Jan 15 09:20:19 rdma-qe-31.rdma.lab.eng.rdu2.redhat.com opensm-launch[15673]: OpenSM 3.3.24 Jan 15 09:20:19 rdma-qe-31.rdma.lab.eng.rdu2.redhat.com OpenSM[15673]: OpenSM 3.3.24 Jan 15 09:20:22 rdma-qe-31.rdma.lab.eng.rdu2.redhat.com opensm-launch[15673]: Using default GUID 0x20af7fffe97b162 Jan 15 09:20:22 rdma-qe-31.rdma.lab.eng.rdu2.redhat.com opensm-launch[15673]: Entering DISCOVERING state Jan 15 09:20:22 rdma-qe-31.rdma.lab.eng.rdu2.redhat.com OpenSM[15673]: Entering DISCOVERING state Jan 15 09:20:26 rdma-qe-31.rdma.lab.eng.rdu2.redhat.com opensm-launch[15673]: ibwarn: [15673] get_abi_version: can't read ABI version from /sys/class/infiniband_mad/abi_version (No such file or directory): is ib_umad module loaded? Jan 15 09:20:26 rdma-qe-31.rdma.lab.eng.rdu2.redhat.com opensm-launch[15673]: Error from osm_opensm_bind (0x2A) Jan 15 09:20:26 rdma-qe-31.rdma.lab.eng.rdu2.redhat.com opensm-launch[15673]: Perhaps another instance of OpenSM is already running Jan 15 09:20:26 rdma-qe-31.rdma.lab.eng.rdu2.redhat.com OpenSM[15673]: Exiting SM Jan 15 09:20:26 rdma-qe-31.rdma.lab.eng.rdu2.redhat.com opensm-launch[15673]: Exiting SM + [23-01-15 09:20:28] SERVICE_RETURN=0 + [23-01-15 09:20:28] return 0 + [23-01-15 09:20:28] '[' 0 -ne 0 ']' + [23-01-15 09:20:28] ifdown bnxt_roce /root/rdma-qa.sh: line 402: ifdown: command not found + [23-01-15 09:20:28] sleep 3 + [23-01-15 09:20:31] ifup bnxt_roce /root/rdma-qa.sh: line 404: ifup: command not found + [23-01-15 09:20:31] local bad=0 + [23-01-15 09:20:31] ping -c3 172.31.40.31 PING 172.31.40.31 (172.31.40.31) 56(84) bytes of data. 64 bytes from 172.31.40.31: icmp_seq=1 ttl=64 time=0.037 ms 64 bytes from 172.31.40.31: icmp_seq=2 ttl=64 time=0.042 ms 64 bytes from 172.31.40.31: icmp_seq=3 ttl=64 time=0.044 ms --- 172.31.40.31 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2078ms rtt min/avg/max/mdev = 0.037/0.041/0.044/0.003 ms + [23-01-15 09:20:33] (( bad+=0 )) + [23-01-15 09:20:33] [[ ! -z rdma-qe-30 ]] + [23-01-15 09:20:33] [[ ! -z rdma-qe-31 ]] + [23-01-15 09:20:33] ping -c3 172.31.40.30 PING 172.31.40.30 (172.31.40.30) 56(84) bytes of data. From 172.31.40.31 icmp_seq=1 Destination Host Unreachable From 172.31.40.31 icmp_seq=2 Destination Host Unreachable From 172.31.40.31 icmp_seq=3 Destination Host Unreachable --- 172.31.40.30 ping statistics --- 3 packets transmitted, 0 received, +3 errors, 100% packet loss, time 2035ms pipe 3 + [23-01-15 09:20:37] (( bad+=1 )) + [23-01-15 09:20:37] '[' 1 -ne 0 ']' + [23-01-15 09:20:37] RQA_assign_static_ip bnxt roce + [23-01-15 09:20:37] '[' 2 -eq 1 ']' + [23-01-15 09:20:37] '[' 2 -eq 2 ']' + [23-01-15 09:20:37] local _RDMA_DRIVER=bnxt + [23-01-15 09:20:37] local _RDMA_NETWORK=roce + [23-01-15 09:20:37] local _DEVICE_ID=bnxt_roce + [23-01-15 09:20:37] _DEVICE_ID=bnxt_roce + [23-01-15 09:20:37] local _net_cfg=/etc/sysconfig/network-scripts/ifcfg-bnxt_roce + [23-01-15 09:20:37] [[ ! -z bnxt_roce ]] ++ [23-01-15 09:20:37] awk -F = '/HWADDR/ {print $2}' /etc/sysconfig/network-scripts/ifcfg-bnxt_roce + [23-01-15 09:20:37] local _DEVICE_MAC=00:0a:f7:97:b1:62 + [23-01-15 09:20:37] [[ -z 00:0a:f7:97:b1:62 ]] + [23-01-15 09:20:37] [[ -z bnxt_roce ]] + [23-01-15 09:20:37] [[ -z 00:0a:f7:97:b1:62 ]] + [23-01-15 09:20:37] '[' -f /root/rdma-functions.sh ']' + [23-01-15 09:20:37] source /root/rdma-functions.sh ++ [23-01-15 09:20:37] RQA_get_rdma_hostname ++ [23-01-15 09:20:37] hostname -A ++ [23-01-15 09:20:37] grep -o 'rdma.*' ++ [23-01-15 09:20:37] awk '{print $1}' ++ [23-01-15 09:20:37] awk -F . '{print $1}' + [23-01-15 09:20:37] local rdma_hostname=rdma-qe-31 ++ [23-01-15 09:20:37] echo rdma-qe-31 ++ [23-01-15 09:20:37] awk -F '[-|.]' '{print $3}' + [23-01-15 09:20:37] local hosttail=10#31 + [23-01-15 09:20:37] local baseint=230 + [23-01-15 09:20:37] case ${rdma_hostname} in + [23-01-15 09:20:37] baseint=0 + [23-01-15 09:20:37] local _creat_net=roce + [23-01-15 09:20:37] _creat_net=roce + [23-01-15 09:20:37] local _creat_intf=bnxt + [23-01-15 09:20:37] INTERFACE=31 + [23-01-15 09:20:37] echo 'Creating static interface: bnxt roce 00:0a:f7:97:b1:62 31' Creating static interface: bnxt roce 00:0a:f7:97:b1:62 31 + [23-01-15 09:20:37] Create_Rdma_Interfaces bnxt roce 00:0a:f7:97:b1:62 31 + [23-01-15 09:20:37] '[' 4 -lt 4 ']' + [23-01-15 09:20:37] CM= + [23-01-15 09:20:37] TYPE=InfiniBand + [23-01-15 09:20:37] HWADDR=hwaddr + [23-01-15 09:20:37] __get_net_from_subnet roce + [23-01-15 09:20:37] case $1 in + [23-01-15 09:20:37] net=40 + [23-01-15 09:20:37] start_net=40 + [23-01-15 09:20:37] case "$2" in + [23-01-15 09:20:37] TYPE=Ethernet + [23-01-15 09:20:37] nets=(${roce_nets[*]}) + [23-01-15 09:20:37] MTU='mtu 9000' + [23-01-15 09:20:37] case "$1" in + [23-01-15 09:20:37] '[' hwaddr = mac -a 4 -lt 6 ']' + [23-01-15 09:20:37] for subnet in ${nets[*]} + [23-01-15 09:20:37] __get_net_from_subnet roce + [23-01-15 09:20:37] case $1 in + [23-01-15 09:20:37] net=40 + [23-01-15 09:20:37] '[' 31 = dhcp ']' + [23-01-15 09:20:37] BP='static 172.31.40.31' + [23-01-15 09:20:37] '[' Ethernet = InfiniBand ']' + [23-01-15 09:20:37] '[' 40 = 40 ']' + [23-01-15 09:20:37] Create_Interface bnxt_roce Ethernet yes hwaddr 00:0a:f7:97:b1:62 static 172.31.40.31 mtu 9000 + [23-01-15 09:20:37] declare bootproto= ipaddr= network= broadcast= stp= + [23-01-15 09:20:37] declare mtu= vlan= pkey= connected_mode= hwaddr= defroute=no + [23-01-15 09:20:37] declare mac= netmask=255.255.255.0 priority= master= + [23-01-15 09:20:37] [[ 9 -lt 3 ]] + [23-01-15 09:20:37] physdev=bnxt_roce + [23-01-15 09:20:37] iftype=Ethernet + [23-01-15 09:20:37] onboot=yes + [23-01-15 09:20:37] shift 3 + [23-01-15 09:20:37] [[ 6 != 0 ]] + [23-01-15 09:20:37] case $1 in + [23-01-15 09:20:37] hwaddr=00:0a:f7:97:b1:62 + [23-01-15 09:20:37] shift 2 + [23-01-15 09:20:37] [[ 4 != 0 ]] + [23-01-15 09:20:37] case $1 in + [23-01-15 09:20:37] bootproto=static + [23-01-15 09:20:37] ipaddr=172.31.40.31 + [23-01-15 09:20:37] shift 2 + [23-01-15 09:20:37] [[ 2 != 0 ]] + [23-01-15 09:20:37] case $1 in + [23-01-15 09:20:37] mtu=9000 + [23-01-15 09:20:37] shift 2 + [23-01-15 09:20:37] [[ 0 != 0 ]] + [23-01-15 09:20:37] '[' Ethernet = infiniband ']' + [23-01-15 09:20:37] '[' rhel = rhel -a '' -lt 6 -a Ethernet = InfiniBand -a -n '' ']' /root/rdma-functions.sh: line 277: [: : integer expression expected + [23-01-15 09:20:37] devname=bnxt_roce + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] output=/etc/sysconfig/network-scripts/ifcfg-bnxt_roce + [23-01-15 09:20:37] '[' rhel = rhel -a '' -lt 6 -a static = dhcp -a no = no ']' /root/rdma-functions.sh: line 292: [: : integer expression expected + [23-01-15 09:20:37] '[' Ethernet = InfiniBand -a static = dhcp ']' + [23-01-15 09:20:37] '[' static = static ']' ++ [23-01-15 09:20:37] echo 172.31.40.31 ++ [23-01-15 09:20:37] cut -f -3 -d . + [23-01-15 09:20:37] network=172.31.40.0 ++ [23-01-15 09:20:37] echo 172.31.40.31 ++ [23-01-15 09:20:37] cut -f -3 -d . + [23-01-15 09:20:37] broadcast=172.31.40.255 + [23-01-15 09:20:37] echo DEVICE=bnxt_roce + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] echo TYPE=Ethernet + [23-01-15 09:20:37] case $iftype in + [23-01-15 09:20:37] case "${master}" in + [23-01-15 09:20:37] '[' Ethernet = InfiniBand ']' + [23-01-15 09:20:37] '[' Ethernet = InfiniBand ']' + [23-01-15 09:20:37] echo ONBOOT=yes + [23-01-15 09:20:37] '[' -n 00:0a:f7:97:b1:62 ']' + [23-01-15 09:20:37] echo HWADDR=00:0a:f7:97:b1:62 + [23-01-15 09:20:37] '[' -n 172.31.40.31 ']' + [23-01-15 09:20:37] echo BOOTPROTO=static + [23-01-15 09:20:37] echo IPADDR=172.31.40.31 + [23-01-15 09:20:37] echo NETMASK=255.255.255.0 + [23-01-15 09:20:37] echo NETWORK=172.31.40.0 + [23-01-15 09:20:37] echo BROADCAST=172.31.40.255 + [23-01-15 09:20:37] echo IPV4_FAILURE_FATAL=yes + [23-01-15 09:20:37] echo PEERDNS=no + [23-01-15 09:20:37] echo IPV6INIT=yes + [23-01-15 09:20:37] echo IPV6_AUTOCONF=yes + [23-01-15 09:20:37] echo IPV6_DEFROUTE=no + [23-01-15 09:20:37] echo IPV6_PEERDNS=no + [23-01-15 09:20:37] echo IPV6_PEERROUTES=yes + [23-01-15 09:20:37] echo IPV6_FAILURE_FATAL=no + [23-01-15 09:20:37] '[' -n 9000 ']' + [23-01-15 09:20:37] echo MTU=9000 + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] echo NAME=bnxt_roce + [23-01-15 09:20:37] '[' Ethernet = InfiniBand ']' + [23-01-15 09:20:37] output=/etc/udev/rules.d/70-persistent-net.rules ++ [23-01-15 09:20:37] echo 00:0a:f7:97:b1:62 + [23-01-15 09:20:37] match_addr=00:0a:f7:97:b1:62 + [23-01-15 09:20:37] '[' '!' -f /etc/udev/rules.d/70-persistent-net.rules ']' + [23-01-15 09:20:37] '[' rhel = rhel -a -gt 5 -a -n 00:0a:f7:97:b1:62 -a -z '' ']' /root/rdma-functions.sh: line 411: [: too many arguments + [23-01-15 09:20:37] '[' rhel = fedora -a -gt 16 -a -n 00:0a:f7:97:b1:62 -a -z '' ']' /root/rdma-functions.sh: line 426: [: too many arguments + [23-01-15 09:20:37] for subnet in ${nets[*]} + [23-01-15 09:20:37] __get_net_from_subnet roce.43 + [23-01-15 09:20:37] case $1 in ++ [23-01-15 09:20:37] echo roce.43 ++ [23-01-15 09:20:37] cut -f 2 -d . + [23-01-15 09:20:37] net=43 + [23-01-15 09:20:37] '[' 31 = dhcp ']' + [23-01-15 09:20:37] BP='static 172.31.43.31' + [23-01-15 09:20:37] '[' Ethernet = InfiniBand ']' + [23-01-15 09:20:37] '[' 43 = 40 ']' + [23-01-15 09:20:37] case $TYPE in + [23-01-15 09:20:37] Create_Interface bnxt_roce Vlan yes static 172.31.43.31 vlan 43 + [23-01-15 09:20:37] declare bootproto= ipaddr= network= broadcast= stp= + [23-01-15 09:20:37] declare mtu= vlan= pkey= connected_mode= hwaddr= defroute=no + [23-01-15 09:20:37] declare mac= netmask=255.255.255.0 priority= master= + [23-01-15 09:20:37] [[ 7 -lt 3 ]] + [23-01-15 09:20:37] physdev=bnxt_roce + [23-01-15 09:20:37] iftype=Vlan + [23-01-15 09:20:37] onboot=yes + [23-01-15 09:20:37] shift 3 + [23-01-15 09:20:37] [[ 4 != 0 ]] + [23-01-15 09:20:37] case $1 in + [23-01-15 09:20:37] bootproto=static + [23-01-15 09:20:37] ipaddr=172.31.43.31 + [23-01-15 09:20:37] shift 2 + [23-01-15 09:20:37] [[ 2 != 0 ]] + [23-01-15 09:20:37] case $1 in + [23-01-15 09:20:37] vlan=43 + [23-01-15 09:20:37] shift 2 + [23-01-15 09:20:37] [[ 0 != 0 ]] + [23-01-15 09:20:37] '[' Vlan = infiniband ']' + [23-01-15 09:20:37] '[' rhel = rhel -a '' -lt 6 -a Vlan = InfiniBand -a -n '' ']' /root/rdma-functions.sh: line 277: [: : integer expression expected + [23-01-15 09:20:37] devname=bnxt_roce + [23-01-15 09:20:37] '[' -n 43 ']' + [23-01-15 09:20:37] devname=bnxt_roce.43 + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] output=/etc/sysconfig/network-scripts/ifcfg-bnxt_roce.43 + [23-01-15 09:20:37] '[' rhel = rhel -a '' -lt 6 -a static = dhcp -a no = no ']' /root/rdma-functions.sh: line 292: [: : integer expression expected + [23-01-15 09:20:37] '[' Vlan = InfiniBand -a static = dhcp ']' + [23-01-15 09:20:37] '[' static = static ']' ++ [23-01-15 09:20:37] echo 172.31.43.31 ++ [23-01-15 09:20:37] cut -f -3 -d . + [23-01-15 09:20:37] network=172.31.43.0 ++ [23-01-15 09:20:37] echo 172.31.43.31 ++ [23-01-15 09:20:37] cut -f -3 -d . + [23-01-15 09:20:37] broadcast=172.31.43.255 + [23-01-15 09:20:37] echo DEVICE=bnxt_roce.43 + [23-01-15 09:20:37] '[' -n 43 ']' + [23-01-15 09:20:37] echo VLAN=yes + [23-01-15 09:20:37] echo VLAN_ID=43 + [23-01-15 09:20:37] '[' 43 -eq 43 ']' + [23-01-15 09:20:37] echo VLAN_EGRESS_PRIORITY_MAP=0:3,1:3,2:3,3:3,4:3,5:3,6:3,7:3 + [23-01-15 09:20:37] '[' 43 -eq 45 ']' + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] echo TYPE=Vlan + [23-01-15 09:20:37] case $iftype in + [23-01-15 09:20:37] '[' Vlan = InfiniBand ']' + [23-01-15 09:20:37] '[' Vlan = InfiniBand ']' + [23-01-15 09:20:37] echo ONBOOT=yes + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] '[' -n 172.31.43.31 ']' + [23-01-15 09:20:37] echo BOOTPROTO=static + [23-01-15 09:20:37] echo IPADDR=172.31.43.31 + [23-01-15 09:20:37] echo NETMASK=255.255.255.0 + [23-01-15 09:20:37] echo NETWORK=172.31.43.0 + [23-01-15 09:20:37] echo BROADCAST=172.31.43.255 + [23-01-15 09:20:37] echo IPV4_FAILURE_FATAL=yes + [23-01-15 09:20:37] echo PEERDNS=no + [23-01-15 09:20:37] echo IPV6INIT=yes + [23-01-15 09:20:37] echo IPV6_AUTOCONF=yes + [23-01-15 09:20:37] echo IPV6_DEFROUTE=no + [23-01-15 09:20:37] echo IPV6_PEERDNS=no + [23-01-15 09:20:37] echo IPV6_PEERROUTES=yes + [23-01-15 09:20:37] echo IPV6_FAILURE_FATAL=no + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] '[' -n '' ']' + [23-01-15 09:20:37] echo NAME=bnxt_roce.43 + [23-01-15 09:20:37] '[' Vlan = InfiniBand ']' + [23-01-15 09:20:37] output=/etc/udev/rules.d/70-persistent-net.rules ++ [23-01-15 09:20:37] echo + [23-01-15 09:20:37] match_addr= + [23-01-15 09:20:37] '[' '!' -f /etc/udev/rules.d/70-persistent-net.rules ']' + [23-01-15 09:20:37] '[' rhel = rhel -a -gt 5 -a -n '' -a -z '' ']' /root/rdma-functions.sh: line 411: [: too many arguments + [23-01-15 09:20:37] '[' rhel = fedora -a -gt 16 -a -n '' -a -z '' ']' /root/rdma-functions.sh: line 426: [: too many arguments + [23-01-15 09:20:37] '[' roce '!=' iw ']' + [23-01-15 09:20:37] Create_Pfc_Egress_Dispatcher bnxt_roce bnxt_roce.43 3 + [23-01-15 09:20:37] '[' rhel = rhel -a '' -lt 6 ']' /root/rdma-functions.sh: line 481: [: : integer expression expected + [23-01-15 09:20:37] prios='3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3' + [23-01-15 09:20:37] user_prios=3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3 + [23-01-15 09:20:37] case $1 in + [23-01-15 09:20:37] cat + [23-01-15 09:20:37] chmod +x /etc/NetworkManager/dispatcher.d/98-bnxt_roce.43-egress.conf + [23-01-15 09:20:37] for subnet in ${nets[*]} + [23-01-15 09:20:37] __get_net_from_subnet roce.45 + [23-01-15 09:20:37] case $1 in ++ [23-01-15 09:20:37] echo roce.45 ++ [23-01-15 09:20:37] cut -f 2 -d . + [23-01-15 09:20:37] net=45 + [23-01-15 09:20:37] '[' 31 = dhcp ']' + [23-01-15 09:20:37] BP='static 172.31.45.31' + [23-01-15 09:20:37] '[' Ethernet = InfiniBand ']' + [23-01-15 09:20:37] '[' 45 = 40 ']' + [23-01-15 09:20:37] case $TYPE in + [23-01-15 09:20:37] Create_Interface bnxt_roce Vlan yes static 172.31.45.31 vlan 45 + [23-01-15 09:20:37] declare bootproto= ipaddr= network= broadcast= stp= + [23-01-15 09:20:37] declare mtu= vlan= pkey= connected_mode= hwaddr= defroute=no + [23-01-15 09:20:37] declare mac= netmask=255.255.255.0 priority= master= + [23-01-15 09:20:37] [[ 7 -lt 3 ]] + [23-01-15 09:20:37] physdev=bnxt_roce + [23-01-15 09:20:37] iftype=Vlan + [23-01-15 09:20:37] onboot=yes + [23-01-15 09:20:37] shift 3 + [23-01-15 09:20:37] [[ 4 != 0 ]] + [23-01-15 09:20:37] case $1 in + [23-01-15 09:20:37] bootproto=static + [23-01-15 09:20:38] ipaddr=172.31.45.31 + [23-01-15 09:20:38] shift 2 + [23-01-15 09:20:38] [[ 2 != 0 ]] + [23-01-15 09:20:38] case $1 in + [23-01-15 09:20:38] vlan=45 + [23-01-15 09:20:38] shift 2 + [23-01-15 09:20:38] [[ 0 != 0 ]] + [23-01-15 09:20:38] '[' Vlan = infiniband ']' + [23-01-15 09:20:38] '[' rhel = rhel -a '' -lt 6 -a Vlan = InfiniBand -a -n '' ']' /root/rdma-functions.sh: line 277: [: : integer expression expected + [23-01-15 09:20:38] devname=bnxt_roce + [23-01-15 09:20:38] '[' -n 45 ']' + [23-01-15 09:20:38] devname=bnxt_roce.45 + [23-01-15 09:20:38] '[' -n '' ']' + [23-01-15 09:20:38] output=/etc/sysconfig/network-scripts/ifcfg-bnxt_roce.45 + [23-01-15 09:20:38] '[' rhel = rhel -a '' -lt 6 -a static = dhcp -a no = no ']' /root/rdma-functions.sh: line 292: [: : integer expression expected + [23-01-15 09:20:38] '[' Vlan = InfiniBand -a static = dhcp ']' + [23-01-15 09:20:38] '[' static = static ']' ++ [23-01-15 09:20:38] echo 172.31.45.31 ++ [23-01-15 09:20:38] cut -f -3 -d . + [23-01-15 09:20:38] network=172.31.45.0 ++ [23-01-15 09:20:38] echo 172.31.45.31 ++ [23-01-15 09:20:38] cut -f -3 -d . + [23-01-15 09:20:38] broadcast=172.31.45.255 + [23-01-15 09:20:38] echo DEVICE=bnxt_roce.45 + [23-01-15 09:20:38] '[' -n 45 ']' + [23-01-15 09:20:38] echo VLAN=yes + [23-01-15 09:20:38] echo VLAN_ID=45 + [23-01-15 09:20:38] '[' 45 -eq 43 ']' + [23-01-15 09:20:38] '[' 45 -eq 45 ']' + [23-01-15 09:20:38] echo VLAN_EGRESS_PRIORITY_MAP=0:5,1:5,2:5,3:5,4:5,5:5,6:5,7:5 + [23-01-15 09:20:38] '[' -n '' ']' + [23-01-15 09:20:38] '[' -n '' ']' + [23-01-15 09:20:38] echo TYPE=Vlan + [23-01-15 09:20:38] case $iftype in + [23-01-15 09:20:38] '[' Vlan = InfiniBand ']' + [23-01-15 09:20:38] '[' Vlan = InfiniBand ']' + [23-01-15 09:20:38] echo ONBOOT=yes + [23-01-15 09:20:38] '[' -n '' ']' + [23-01-15 09:20:38] '[' -n 172.31.45.31 ']' + [23-01-15 09:20:38] echo BOOTPROTO=static + [23-01-15 09:20:38] echo IPADDR=172.31.45.31 + [23-01-15 09:20:38] echo NETMASK=255.255.255.0 + [23-01-15 09:20:38] echo NETWORK=172.31.45.0 + [23-01-15 09:20:38] echo BROADCAST=172.31.45.255 + [23-01-15 09:20:38] echo IPV4_FAILURE_FATAL=yes + [23-01-15 09:20:38] echo PEERDNS=no + [23-01-15 09:20:38] echo IPV6INIT=yes + [23-01-15 09:20:38] echo IPV6_AUTOCONF=yes + [23-01-15 09:20:38] echo IPV6_DEFROUTE=no + [23-01-15 09:20:38] echo IPV6_PEERDNS=no + [23-01-15 09:20:38] echo IPV6_PEERROUTES=yes + [23-01-15 09:20:38] echo IPV6_FAILURE_FATAL=no + [23-01-15 09:20:38] '[' -n '' ']' + [23-01-15 09:20:38] '[' -n '' ']' + [23-01-15 09:20:38] echo NAME=bnxt_roce.45 + [23-01-15 09:20:38] '[' Vlan = InfiniBand ']' + [23-01-15 09:20:38] output=/etc/udev/rules.d/70-persistent-net.rules ++ [23-01-15 09:20:38] echo + [23-01-15 09:20:38] match_addr= + [23-01-15 09:20:38] '[' '!' -f /etc/udev/rules.d/70-persistent-net.rules ']' + [23-01-15 09:20:38] '[' rhel = rhel -a -gt 5 -a -n '' -a -z '' ']' /root/rdma-functions.sh: line 411: [: too many arguments + [23-01-15 09:20:38] '[' rhel = fedora -a -gt 16 -a -n '' -a -z '' ']' /root/rdma-functions.sh: line 426: [: too many arguments + [23-01-15 09:20:38] '[' roce '!=' iw ']' + [23-01-15 09:20:38] Create_Pfc_Egress_Dispatcher bnxt_roce bnxt_roce.45 5 + [23-01-15 09:20:38] '[' rhel = rhel -a '' -lt 6 ']' /root/rdma-functions.sh: line 481: [: : integer expression expected + [23-01-15 09:20:38] prios='5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5' + [23-01-15 09:20:38] user_prios=5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 + [23-01-15 09:20:38] case $1 in + [23-01-15 09:20:38] cat + [23-01-15 09:20:38] chmod +x /etc/NetworkManager/dispatcher.d/98-bnxt_roce.45-egress.conf + [23-01-15 09:20:38] shift 4 + [23-01-15 09:20:38] '[' 0 -ge 2 ']' + [23-01-15 09:20:38] [[ bnxt == *\m\l\x\5* ]] ++ [23-01-15 09:20:38] ls /etc/sysconfig/network-scripts/ifcfg-bnxt_roce /etc/sysconfig/network-scripts/ifcfg-bnxt_roce.43 /etc/sysconfig/network-scripts/ifcfg-bnxt_roce.45 ++ [23-01-15 09:20:38] awk -F - '{print $(NF)}' + [23-01-15 09:20:38] for d in $(ls ${_net_cfg}* | awk -F "-" '{print $(NF)}') + [23-01-15 09:20:38] ifdown bnxt_roce /root/rdma-qa.sh: line 504: ifdown: command not found + [23-01-15 09:20:38] sleep 3 + [23-01-15 09:20:41] ifup bnxt_roce /root/rdma-qa.sh: line 506: ifup: command not found + [23-01-15 09:20:41] for d in $(ls ${_net_cfg}* | awk -F "-" '{print $(NF)}') + [23-01-15 09:20:41] ifdown bnxt_roce.43 /root/rdma-qa.sh: line 504: ifdown: command not found + [23-01-15 09:20:41] sleep 3 + [23-01-15 09:20:44] ifup bnxt_roce.43 /root/rdma-qa.sh: line 506: ifup: command not found + [23-01-15 09:20:44] for d in $(ls ${_net_cfg}* | awk -F "-" '{print $(NF)}') + [23-01-15 09:20:44] ifdown bnxt_roce.45 /root/rdma-qa.sh: line 504: ifdown: command not found + [23-01-15 09:20:44] sleep 3 + [23-01-15 09:20:47] ifup bnxt_roce.45 /root/rdma-qa.sh: line 506: ifup: command not found + [23-01-15 09:20:47] [[ bnxt == \s\i\w ]] + [23-01-15 09:20:47] sleep 5 ++ [23-01-15 09:20:52] RQA_get_my_ipv4 bnxt_roce ++ [23-01-15 09:20:52] local IPv4= ++ [23-01-15 09:20:52] '[' 1 -ne 1 ']' +++ [23-01-15 09:20:52] ip addr show dev bnxt_roce +++ [23-01-15 09:20:52] sed '-es/^.*inet \([^ ]*\)\/.*$/\1/;t;d' ++ [23-01-15 09:20:52] IPv4=172.31.40.31 ++ [23-01-15 09:20:52] [[ 172.31.40.31 == '' ]] ++ [23-01-15 09:20:52] echo 172.31.40.31 ++ [23-01-15 09:20:52] return 0 + [23-01-15 09:20:52] RDMA_IPV4=172.31.40.31 ++ [23-01-15 09:20:52] RQA_get_my_ipv6 bnxt_roce ++ [23-01-15 09:20:52] '[' 1 -ne 1 ']' ++ [23-01-15 09:20:52] ip addr show dev bnxt_roce ++ [23-01-15 09:20:52] sed '-es/^.*inet6 \([^ ]*\)\/.*$/\1/;t;d' + [23-01-15 09:20:52] RDMA_IPV6=fe80::20a:f7ff:fe97:b162 + [23-01-15 09:20:52] [[ bnxt == \s\i\w ]] + [23-01-15 09:20:52] [[ bnxt == \r\x\e ]] + [23-01-15 09:20:52] grep -i fedora /etc/redhat-release + [23-01-15 09:20:52] ping -c 3 172.31.40.31 PING 172.31.40.31 (172.31.40.31) 56(84) bytes of data. 64 bytes from 172.31.40.31: icmp_seq=1 ttl=64 time=0.035 ms 64 bytes from 172.31.40.31: icmp_seq=2 ttl=64 time=0.056 ms 64 bytes from 172.31.40.31: icmp_seq=3 ttl=64 time=0.038 ms --- 172.31.40.31 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2070ms rtt min/avg/max/mdev = 0.035/0.043/0.056/0.009 ms + [23-01-15 09:20:54] RQA_check_result -r 0 -t 'ping 172.31.40.31' + [23-01-15 09:20:54] local test_pass=0 + [23-01-15 09:20:54] local test_skip=777 + [23-01-15 09:20:54] test 4 -gt 0 + [23-01-15 09:20:54] case $1 in + [23-01-15 09:20:54] local rc=0 + [23-01-15 09:20:54] shift + [23-01-15 09:20:54] shift + [23-01-15 09:20:54] test 2 -gt 0 + [23-01-15 09:20:54] case $1 in + [23-01-15 09:20:54] local 'msg=ping 172.31.40.31' + [23-01-15 09:20:54] shift + [23-01-15 09:20:54] shift + [23-01-15 09:20:54] test 0 -gt 0 + [23-01-15 09:20:54] '[' -z 0 -o -z 'ping 172.31.40.31' ']' + [23-01-15 09:20:54] '[' -z /tmp/tmp.mXlwpDN8bq/results_env_setup.txt ']' + [23-01-15 09:20:54] '[' -z /tmp/tmp.mXlwpDN8bq/results_env_setup.txt ']' + [23-01-15 09:20:54] '[' 0 -eq 0 ']' + [23-01-15 09:20:54] local test_result=PASS + [23-01-15 09:20:54] printf '%10s | %6s | %s\n' PASS 0 'ping 172.31.40.31' + [23-01-15 09:20:54] set +x --- - TEST RESULT FOR env_setup - Test: ping 172.31.40.31 - Result: PASS - Return: 0 --- + [23-01-15 09:20:54] standalone + [23-01-15 09:20:54] echo -en '\n# /kernel/infiniband/env_setup variables\n' + [23-01-15 09:20:54] [[ -s /root/py_parse_out.txt ]] + [23-01-15 09:20:54] grep export /root/py_parse_out.txt + [23-01-15 09:20:54] cat + [23-01-15 09:20:54] echo 'export RDMA_IPV4="172.31.40.31"' + [23-01-15 09:20:54] echo 'export RDMA_IPV6="fe80::20a:f7ff:fe97:b162"' + [23-01-15 09:20:54] source /root/.bashrc + [23-01-15 09:20:54] RQA_check_result -r 0 -t 'Source .bashrc after updates' + [23-01-15 09:20:54] local test_pass=0 + [23-01-15 09:20:54] local test_skip=777 + [23-01-15 09:20:54] test 4 -gt 0 + [23-01-15 09:20:54] case $1 in + [23-01-15 09:20:54] local rc=0 + [23-01-15 09:20:54] shift + [23-01-15 09:20:54] shift + [23-01-15 09:20:54] test 2 -gt 0 + [23-01-15 09:20:54] case $1 in + [23-01-15 09:20:54] local 'msg=Source .bashrc after updates' + [23-01-15 09:20:54] shift + [23-01-15 09:20:54] shift + [23-01-15 09:20:54] test 0 -gt 0 + [23-01-15 09:20:54] '[' -z 0 -o -z 'Source .bashrc after updates' ']' + [23-01-15 09:20:54] '[' -z /tmp/tmp.mXlwpDN8bq/results_env_setup.txt ']' + [23-01-15 09:20:54] '[' -z /tmp/tmp.mXlwpDN8bq/results_env_setup.txt ']' + [23-01-15 09:20:54] '[' 0 -eq 0 ']' + [23-01-15 09:20:54] local test_result=PASS + [23-01-15 09:20:54] printf '%10s | %6s | %s\n' PASS 0 'Source .bashrc after updates' + [23-01-15 09:20:54] set +x --- - TEST RESULT FOR env_setup - Test: Source .bashrc after updates - Result: PASS - Return: 0 --- + [23-01-15 09:20:54] RQA_overall_result + [23-01-15 09:20:54] source /root/known-issues.sh + [23-01-15 09:20:59] '[' -z /tmp/tmp.mXlwpDN8bq/results_env_setup.txt ']' + [23-01-15 09:20:59] '[' -z /tmp/tmp.mXlwpDN8bq/results_env_setup.txt ']' + [23-01-15 09:20:59] local overall_result=PASS + [23-01-15 09:20:59] echo 'Checking for failures and known issues:' + [23-01-15 09:20:59] grep FAIL /tmp/tmp.mXlwpDN8bq/results_env_setup.txt + [23-01-15 09:20:59] echo ' no test failures' + [23-01-15 09:20:59] echo '' + [23-01-15 09:20:59] report_result /kernel/infiniband/env_setup/standalone PASS 0 + [23-01-15 09:20:59] OUTPUTFILE=/mnt/testarea/tmp.mSwgTm + [23-01-15 09:20:59] rstrnt-report-result /kernel/infiniband/env_setup/standalone PASS 0 ** /kernel/infiniband/env_setup/standalone PASS Score:0 Uploading resultoutputfile.log .done + [23-01-15 09:21:01] rhts-submit-log -l /tmp/tmp.mXlwpDN8bq/results_env_setup.txt Uploading results_env_setup.txt .done + [23-01-15 09:21:01] __RQA_unset_test_result_file + [23-01-15 09:21:01] cat /tmp/tmp.mXlwpDN8bq/results_env_setup.txt env_setup test results on rdma-qe-31/rdma-qe-30 & Beaker job J:7432005: 5.14.0-235.rt14.236.1639_747671795.el9.x86_64, rdma-core-41.0-3.el9, bnxt, roce, BCM57414 & bnxt_re0 Result | Status | Test ---------+--------+------------------------------------ PASS | 0 | Source python parser output PASS | 0 | ping 172.31.40.31 PASS | 0 | Source .bashrc after updates Checking for failures and known issues: no test failures + [23-01-15 09:21:01] export TEST_RESULT_FILE= + [23-01-15 09:21:01] TEST_RESULT_FILE= + [23-01-15 09:21:01] unset TEST_RESULT_FILE + [23-01-15 09:21:01] sed -i /TEST_RESULT_FILE/d /etc/environment + [23-01-15 09:21:01] echo 'TEST_RESULT_FILE=""' + [23-01-15 09:21:01] source /etc/environment ++ [23-01-15 09:21:01] TEST_RESULT_FILE= + [23-01-15 09:21:01] return 0 + [23-01-15 09:21:01] hostname -A + [23-01-15 09:21:01] grep rdma-qe-30 + [23-01-15 09:21:01] hostname -A + [23-01-15 09:21:01] grep rdma-qe-31 + [23-01-15 09:21:01] echo '------- server start test -------' ------- server start test ------- + [23-01-15 09:21:01] export result=PASS + [23-01-15 09:21:01] result=PASS + [23-01-15 09:21:01] export TEST=/kernel/infiniband/env_setup/standalone/server + [23-01-15 09:21:01] TEST=/kernel/infiniband/env_setup/standalone/server + [23-01-15 09:21:01] server + [23-01-15 09:21:01] [[ bnxt == \s\i\w ]] + [23-01-15 09:21:01] rhts_sync_block -s ES_CLIENT_READY_-roce-0 rdma-qe-30 /usr/bin/rhts_sync_block -s ES_CLIENT_READY_-roce-0 rdma-qe-30 -- Blocking state(s) = 12_ES_CLIENT_READY_-roce-0 Failed to connect to remote server rdma-qe-30: No route to host Retrying rstrnt-sync-block for state 12_ES_CLIENT_READY_-roce-0 Failed to connect to remote server rdma-qe-30: No route to host Retrying rstrnt-sync-block for state 12_ES_CLIENT_READY_-roce-0 Failed to connect to remote server rdma-qe-30: Connection refused Retrying rstrnt-sync-block for state 12_ES_CLIENT_READY_-roce-0 Failed to connect to remote server rdma-qe-30: Connection refused Retrying rstrnt-sync-block for state 12_ES_CLIENT_READY_-roce-0 Failed to connect to remote server rdma-qe-30: Connection refused Retrying rstrnt-sync-block for state 12_ES_CLIENT_READY_-roce-0 + [23-01-15 09:26:07] rhts_sync_set -s ES_SERVER_READY_-roce-0 /usr/bin/rhts_sync_set -s ES_SERVER_READY_-roce-0 -- State set = 12_ES_SERVER_READY_-roce-0 ++ [23-01-15 09:26:07] echo rdma-qe-30 ++ [23-01-15 09:26:07] awk -F . '{print $1}' + [23-01-15 09:26:07] cli_ssh=rdma-qe-30 ++ [23-01-15 09:26:07] ssh rdma-qe-30 echo '${RDMA_DRIVER}' + [23-01-15 09:26:08] CLIENT_DRIVER=bnxt ++ [23-01-15 09:26:08] ssh rdma-qe-30 echo '${RDMA_NETWORK}' + [23-01-15 09:26:08] CLIENT_NETWORK=roce ++ [23-01-15 09:26:08] ssh rdma-qe-30 echo '${HCA_ID}' + [23-01-15 09:26:09] CLIENT_HCA_ID=bnxt_re0 ++ [23-01-15 09:26:09] ssh rdma-qe-30 echo '${HCA_ABRV}' + [23-01-15 09:26:09] CLIENT_HCA_ABRV=BCM57414 ++ [23-01-15 09:26:09] ssh rdma-qe-30 echo '${DEVICE_ID}' + [23-01-15 09:26:09] CLIENT_DEVICE_ID=bnxt_roce ++ [23-01-15 09:26:09] ssh rdma-qe-30 echo '${DEVICE_MAC}' + [23-01-15 09:26:10] CLIENT_DEVICE_MAC=00:0a:f7:a6:89:70 ++ [23-01-15 09:26:10] ssh rdma-qe-30 echo '${DEVICE_PORT}' + [23-01-15 09:26:10] CLIENT_DEVICE_PORT=1 ++ [23-01-15 09:26:10] ssh rdma-qe-30 echo '${RDMA_IPV4}' + [23-01-15 09:26:10] CLIENT_IPV4=172.31.40.30 ++ [23-01-15 09:26:10] ssh rdma-qe-30 echo '${RDMA_IPV6}' + [23-01-15 09:26:11] CLIENT_IPV6=fe80::20a:f7ff:fea6:8970 + [23-01-15 09:26:11] '[' -z bnxt ']' + [23-01-15 09:26:11] '[' -z roce ']' + [23-01-15 09:26:11] '[' -z bnxt_re0 ']' + [23-01-15 09:26:11] '[' -z BCM57414 ']' + [23-01-15 09:26:11] '[' -z bnxt_roce ']' + [23-01-15 09:26:11] '[' -z 00:0a:f7:a6:89:70 ']' + [23-01-15 09:26:11] '[' -z 1 ']' + [23-01-15 09:26:11] '[' -z 172.31.40.30 ']' + [23-01-15 09:26:11] '[' -z fe80::20a:f7ff:fea6:8970 ']' + [23-01-15 09:26:11] echo 'export CLIENT_DRIVER="bnxt"' + [23-01-15 09:26:11] echo 'export CLIENT_NETWORK="roce"' + [23-01-15 09:26:11] echo 'export CLIENT_HCA_ID="bnxt_re0"' + [23-01-15 09:26:11] echo 'export CLIENT_HCA_ABRV="BCM57414"' + [23-01-15 09:26:11] echo 'export CLIENT_DEVICE_ID="bnxt_roce"' + [23-01-15 09:26:11] echo 'export CLIENT_DEVICE_MAC="00:0a:f7:a6:89:70"' + [23-01-15 09:26:11] echo 'export CLIENT_DEVICE_PORT="1"' + [23-01-15 09:26:11] echo 'export CLIENT_IPV4="172.31.40.30"' + [23-01-15 09:26:11] echo 'export CLIENT_IPV6="fe80::20a:f7ff:fea6:8970"' + [23-01-15 09:26:11] echo 'export SERVER_DRIVER="bnxt"' + [23-01-15 09:26:11] echo 'export SERVER_NETWORK="roce"' + [23-01-15 09:26:11] echo 'export SERVER_HCA_ID="bnxt_re0"' + [23-01-15 09:26:11] echo 'export SERVER_HCA_ABRV="BCM57414"' + [23-01-15 09:26:11] echo 'export SERVER_DEVICE_ID="bnxt_roce"' + [23-01-15 09:26:11] echo 'export SERVER_DEVICE_MAC="00:0a:f7:97:b1:62"' + [23-01-15 09:26:11] echo 'export SERVER_DEVICE_PORT="1"' + [23-01-15 09:26:11] echo 'export SERVER_IPV4="172.31.40.31"' + [23-01-15 09:26:11] echo 'export SERVER_IPV6="fe80::20a:f7ff:fe97:b162"' + [23-01-15 09:26:11] echo 'export REMOTE_HOST="rdma-qe-30"' + [23-01-15 09:26:11] source /root/.bashrc + [23-01-15 09:26:11] RQA_check_result -r 0 -t 'source .bashrc in server function' + [23-01-15 09:26:11] local test_pass=0 + [23-01-15 09:26:11] local test_skip=777 + [23-01-15 09:26:11] test 4 -gt 0 + [23-01-15 09:26:11] case $1 in + [23-01-15 09:26:11] local rc=0 + [23-01-15 09:26:11] shift + [23-01-15 09:26:11] shift + [23-01-15 09:26:11] test 2 -gt 0 + [23-01-15 09:26:11] case $1 in + [23-01-15 09:26:11] local 'msg=source .bashrc in server function' + [23-01-15 09:26:11] shift + [23-01-15 09:26:11] shift + [23-01-15 09:26:11] test 0 -gt 0 + [23-01-15 09:26:11] '[' -z 0 -o -z 'source .bashrc in server function' ']' + [23-01-15 09:26:11] '[' -z '' ']' + [23-01-15 09:26:11] source /etc/environment ++ [23-01-15 09:26:11] TEST_RESULT_FILE= + [23-01-15 09:26:11] '[' -z '' ']' + [23-01-15 09:26:11] __RQA_create_test_result_file ++ [23-01-15 09:26:11] rpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}' rdma-core + [23-01-15 09:26:11] local rdma_version=rdma-core-41.0-3.el9 ++ [23-01-15 09:26:11] echo /kernel/infiniband/env_setup/standalone/server ++ [23-01-15 09:26:11] sed -r 's/\/kernel\/infiniband\///;s/\/server//;s/\/client//;s/\/standalone//' + [23-01-15 09:26:11] TEST_NAME=env_setup + [23-01-15 09:26:11] [[ ! -z rdma-qe-31 ]] + [23-01-15 09:26:11] [[ ! -z rdma-qe-30 ]] + [23-01-15 09:26:11] hosts_list=rdma-qe-31/rdma-qe-30 + [23-01-15 09:26:11] local _bkrjb= ++ [23-01-15 09:26:11] grep JOBID= /etc/motd ++ [23-01-15 09:26:11] tr -d ' ' ++ [23-01-15 09:26:11] sed s/JOBID=/J:/g ++ [23-01-15 09:26:11] head -1 + [23-01-15 09:26:11] _bkrjb=J:7432005 + [23-01-15 09:26:11] [[ ! -z J:7432005 ]] + [23-01-15 09:26:11] _bkrjb=' & Beaker job J:7432005' ++ [23-01-15 09:26:11] echo results_env_setup.txt ++ [23-01-15 09:26:11] sed 's/\//-/g' + [23-01-15 09:26:11] local result_filename=results_env_setup.txt ++ [23-01-15 09:26:11] mktemp -d + [23-01-15 09:26:11] TEST_RESULT_FILE=/tmp/tmp.zyluXGnOLD/results_env_setup.txt + [23-01-15 09:26:11] echo 'env_setup test results on rdma-qe-31/rdma-qe-30 & Beaker job J:7432005:' ++ [23-01-15 09:26:11] uname -r + [23-01-15 09:26:11] echo '5.14.0-235.rt14.236.1639_747671795.el9.x86_64, rdma-core-41.0-3.el9, bnxt, roce, BCM57414 & bnxt_re0' + [23-01-15 09:26:11] echo ' Result | Status | Test' + [23-01-15 09:26:11] echo ' ---------+--------+------------------------------------' + [23-01-15 09:26:11] sed -i /TEST_RESULT_FILE/d /etc/environment + [23-01-15 09:26:11] echo TEST_RESULT_FILE=/tmp/tmp.zyluXGnOLD/results_env_setup.txt + [23-01-15 09:26:11] source /etc/environment ++ [23-01-15 09:26:11] TEST_RESULT_FILE=/tmp/tmp.zyluXGnOLD/results_env_setup.txt + [23-01-15 09:26:11] '[' 0 -eq 0 ']' + [23-01-15 09:26:11] local test_result=PASS + [23-01-15 09:26:11] printf '%10s | %6s | %s\n' PASS 0 'source .bashrc in server function' + [23-01-15 09:26:11] set +x --- - TEST RESULT FOR env_setup - Test: source .bashrc in server function - Result: PASS - Return: 0 --- + [23-01-15 09:26:11] rhts_sync_set -s ES_SERVER_DONE_-roce-0 /usr/bin/rhts_sync_set -s ES_SERVER_DONE_-roce-0 -- State set = 12_ES_SERVER_DONE_-roce-0 + [23-01-15 09:26:11] echo '--- server finishes.' --- server finishes. + [23-01-15 09:26:11] rhts_sync_block -s ES_CLIENT_DONE_-roce-0 rdma-qe-30 /usr/bin/rhts_sync_block -s ES_CLIENT_DONE_-roce-0 rdma-qe-30 -- Blocking state(s) = 12_ES_CLIENT_DONE_-roce-0 Server rdma-qe-30 not reported state 12_ES_CLIENT_DONE_-roce-0 for Multihost Sync rcvd 0 bytes, ret 0 [0, 0] Retrying rstrnt-sync-block for state 12_ES_CLIENT_DONE_-roce-0 + [23-01-15 09:27:21] RQA_overall_result + [23-01-15 09:27:21] source /root/known-issues.sh + [23-01-15 09:27:26] '[' -z /tmp/tmp.zyluXGnOLD/results_env_setup.txt ']' + [23-01-15 09:27:26] '[' -z /tmp/tmp.zyluXGnOLD/results_env_setup.txt ']' + [23-01-15 09:27:26] local overall_result=PASS + [23-01-15 09:27:26] echo 'Checking for failures and known issues:' + [23-01-15 09:27:26] grep FAIL /tmp/tmp.zyluXGnOLD/results_env_setup.txt + [23-01-15 09:27:26] echo ' no test failures' + [23-01-15 09:27:26] echo '' + [23-01-15 09:27:26] report_result /kernel/infiniband/env_setup/standalone/server PASS 0 + [23-01-15 09:27:26] OUTPUTFILE=/mnt/testarea/tmp.mSwgTm + [23-01-15 09:27:26] rstrnt-report-result /kernel/infiniband/env_setup/standalone/server PASS 0 ** /kernel/infiniband/env_setup/standalone/server PASS Score:0 Uploading resultoutputfile.log