+ [22-09-24 02:31:56] '[' multi == multi ']' + [22-09-24 02:31:56] echo 'Clients: rdma-qe-12.rdma.lab.eng.rdu2.redhat.com' Clients: rdma-qe-12.rdma.lab.eng.rdu2.redhat.com + [22-09-24 02:31:56] echo 'Servers: rdma-dev-13.rdma.lab.eng.rdu2.redhat.com' Servers: rdma-dev-13.rdma.lab.eng.rdu2.redhat.com + [22-09-24 02:31:56] RQA_system_info_for_debug + [22-09-24 02:31:56] grep -i distro /etc/motd + [22-09-24 02:31:56] tr -d ' ' DISTRO= DISTRO= + [22-09-24 02:31:56] cat /etc/redhat-release CentOS Stream release 9 + [22-09-24 02:31:56] uname -a Linux rdma-qe-12.rdma.lab.eng.rdu2.redhat.com 5.14.0-168.mr1359_220923_1526.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Sep 23 15:37:57 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux + [22-09-24 02:31:56] cat /proc/cmdline BOOT_IMAGE=(hd0,msdos1)/vmlinuz-5.14.0-168.mr1359_220923_1526.el9.x86_64 root=UUID=c34207f0-14a3-4ba4-af6a-fed8d898cbdd ro intel_idle.max_cstate=0 intremap=no_x2apic_optout processor.max_cstate=0 rd.driver.blacklist=csiostor console=tty0 rd_NO_PLYMOUTH resume=UUID=c632c698-db09-462d-a5e8-f080932a3eaa console=ttyS1,115200n81 crashkernel=1G-4G:384M,4G-16G:512M,16G-64G:1G,64G-128G:2G,128G-:4G + [22-09-24 02:31:56] rpm -q rdma-core linux-firmware rdma-core-41.0-3.el9.x86_64 linux-firmware-20220708-127.el9.noarch + [22-09-24 02:31:56] tail /sys/class/infiniband/cxgb4_0/fw_ver 1.26.6.0 + [22-09-24 02:31:56] lspci + [22-09-24 02:31:56] grep -i -e ethernet -e infiniband -e omni -e ConnectX 03:00.0 Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5717 Gigabit Ethernet PCIe (rev 10) 03:00.1 Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5717 Gigabit Ethernet PCIe (rev 10) 05:00.0 Ethernet controller: Chelsio Communications Inc T520-CR Unified Wire Ethernet Controller 05:00.1 Ethernet controller: Chelsio Communications Inc T520-CR Unified Wire Ethernet Controller 05:00.2 Ethernet controller: Chelsio Communications Inc T520-CR Unified Wire Ethernet Controller 05:00.3 Ethernet controller: Chelsio Communications Inc T520-CR Unified Wire Ethernet Controller 05:00.4 Ethernet controller: Chelsio Communications Inc T520-CR Unified Wire Ethernet Controller + [22-09-24 02:31:56] lscpu Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Address sizes: 36 bits physical, 48 bits virtual Byte Order: Little Endian CPU(s): 8 On-line CPU(s) list: 0-7 Vendor ID: GenuineIntel BIOS Vendor ID: Intel Model name: Intel(R) Xeon(R) CPU E3-1240 V2 @ 3.40GHz BIOS Model name: Intel(R) Xeon(R) CPU E3-1240 V2 @ 3.40GHz CPU family: 6 Model: 58 Thread(s) per core: 2 Core(s) per socket: 4 Socket(s): 1 Stepping: 9 CPU max MHz: 3800.0000 CPU min MHz: 1600.0000 BogoMIPS: 6784.38 Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm cpuid_fault epb pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase smep erms xsaveopt dtherm ida arat pln pts md_clear flush_l1d Virtualization: VT-x L1d cache: 128 KiB (4 instances) L1i cache: 128 KiB (4 instances) L2 cache: 1 MiB (4 instances) L3 cache: 8 MiB (1 instance) NUMA node(s): 1 NUMA node0 CPU(s): 0-7 Vulnerability Itlb multihit: KVM: Mitigation: VMX disabled Vulnerability L1tf: Mitigation; PTE Inversion; VMX conditional cache flushes, SMT vulnerable Vulnerability Mds: Mitigation; Clear CPU buffers; SMT vulnerable Vulnerability Meltdown: Mitigation; PTI Vulnerability Mmio stale data: Not affected Vulnerability Retbleed: Not affected Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization Vulnerability Spectre v2: Mitigation; Retpolines, IBPB conditional, IBRS_FW, STIBP conditional, RSB filling, PBRSB-eIBRS Not affected Vulnerability Srbds: Vulnerable: No microcode Vulnerability Tsx async abort: Not affected + [22-09-24 02:31:56] ibstat + [22-09-24 02:31:56] ibstatus Infiniband device 'cxgb4_0' port 1 status: default gid: 0007:432f:2040:0000:0000:0000:0000:0000 base lid: 0x0 sm lid: 0x0 state: 4: ACTIVE phys state: 5: LinkUp rate: 10 Gb/sec (1X FDR10) link_layer: Ethernet Infiniband device 'cxgb4_0' port 2 status: default gid: 0007:432f:2048:0000:0000:0000:0000:0000 base lid: 0x0 sm lid: 0x0 state: 1: DOWN phys state: 3: Disabled rate: 2.5 Gb/sec (1X SDR) link_layer: Ethernet + [22-09-24 02:31:56] /usr/bin/ibv_devinfo -v hca_id: cxgb4_0 transport: iWARP (1) fw_ver: 1.26.6.0 node_guid: 0207:43ff:fe2f:2040 sys_image_guid: 0207:43ff:fe2f:2040 vendor_id: 0x1425 vendor_part_id: 21505 hw_ver: 0x0 board_id: 1425.5401 phys_port_cnt: 2 max_mr_size: 0xffffffffffffffff page_size_cap: 0xffff000 max_qp: 12288 max_qp_wr: 8190 device_cap_flags: 0x00228000 MEM_WINDOW MEM_MGT_EXTENSIONS Unknown flags: 0x8000 max_sge: 4 max_sge_rd: 1 max_cq: 24576 max_cqe: 65518 max_mr: 381300 max_pd: 65536 max_qp_rd_atom: 21 max_ee_rd_atom: 0 max_res_rd_atom: 387072 max_qp_init_rd_atom: 21 max_ee_init_rd_atom: 0 atomic_cap: ATOMIC_NONE (0) max_ee: 0 max_rdd: 0 max_mw: 0 max_raw_ipv6_qp: 0 max_raw_ethy_qp: 0 max_mcast_grp: 0 max_mcast_qp_attach: 0 max_total_mcast_qp_attach: 0 max_ah: 0 max_fmr: 0 max_srq: 0 max_pkeys: 0 local_ca_ack_delay: 0 general_odp_caps: rc_odp_caps: NO SUPPORT uc_odp_caps: NO SUPPORT ud_odp_caps: NO SUPPORT xrc_odp_caps: NO SUPPORT completion_timestamp_mask not supported core clock not supported device_cap_flags_ex: 0x228000 tso_caps: max_tso: 0 rss_caps: max_rwq_indirection_tables: 0 max_rwq_indirection_table_size: 0 rx_hash_function: 0x0 rx_hash_fields_mask: 0x0 max_wq_type_rq: 0 packet_pacing_caps: qp_rate_limit_min: 0kbps qp_rate_limit_max: 0kbps tag matching not supported num_comp_vectors: 8 port: 1 state: PORT_ACTIVE (4) max_mtu: 4096 (5) active_mtu: 4096 (5) sm_lid: 0 port_lid: 0 port_lmc: 0x00 link_layer: Ethernet max_msg_sz: 0xffffffff port_cap_flags: 0x009f0000 port_cap_flags2: 0x0000 max_vl_num: invalid value (0) bad_pkey_cntr: 0x0 qkey_viol_cntr: 0x0 sm_sl: 0 pkey_tbl_len: 0 gid_tbl_len: 1 subnet_timeout: 0 init_type_reply: 0 active_width: 1X (1) active_speed: 10.0 Gbps (8) port: 2 state: PORT_DOWN (1) max_mtu: 4096 (5) active_mtu: 1024 (3) sm_lid: 0 port_lid: 0 port_lmc: 0x00 link_layer: Ethernet max_msg_sz: 0xffffffff port_cap_flags: 0x009f0000 port_cap_flags2: 0x0000 max_vl_num: invalid value (0) bad_pkey_cntr: 0x0 qkey_viol_cntr: 0x0 sm_sl: 0 pkey_tbl_len: 0 gid_tbl_len: 1 subnet_timeout: 0 init_type_reply: 0 active_width: 1X (1) active_speed: 2.5 Gbps (1) + [22-09-24 02:31:56] iblinkinfo ibwarn: [9733] get_abi_version: can't read ABI version from /sys/class/infiniband_mad/abi_version (No such file or directory): is ib_umad module loaded? ibwarn: [9733] mad_rpc_open_port: can't open UMAD port ((null):0) Failed to open (null) port 0 + [22-09-24 02:31:56] [[ '' == *\o\p\a* ]] + [22-09-24 02:31:56] ip addr show 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: lom_1: mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 34:64:a9:95:c9:3c brd ff:ff:ff:ff:ff:ff altname enp3s0f0 altname eno1 inet 10.1.217.67/24 brd 10.1.217.255 scope global dynamic noprefixroute lom_1 valid_lft 86283sec preferred_lft 86283sec inet6 2620:52:0:1d9:3664:a9ff:fe95:c93c/64 scope global dynamic noprefixroute valid_lft 2591966sec preferred_lft 604766sec inet6 fe80::3664:a9ff:fe95:c93c/64 scope link noprefixroute valid_lft forever preferred_lft forever 3: lom_2: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 34:64:a9:95:c9:3d brd ff:ff:ff:ff:ff:ff altname enp3s0f1 altname eno2 4: cxgb4_iw: mtu 9000 qdisc mq state UP group default qlen 1000 link/ether 00:07:43:2f:20:40 brd ff:ff:ff:ff:ff:ff altname enp5s0f4 altname ens2f4 inet 172.31.50.12/24 brd 172.31.50.255 scope global dynamic noprefixroute cxgb4_iw valid_lft 3475sec preferred_lft 3475sec inet6 fe80::207:43ff:fe2f:2040/64 scope link noprefixroute valid_lft forever preferred_lft forever 5: cxgb4_off: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 00:07:43:2f:20:48 brd ff:ff:ff:ff:ff:ff altname enp5s0f4d1 altname ens2f4d1 6: cxgb4_iw.51@cxgb4_iw: mtu 9000 qdisc noqueue state UP group default qlen 1000 link/ether 00:07:43:2f:20:40 brd ff:ff:ff:ff:ff:ff inet 172.31.51.12/24 brd 172.31.51.255 scope global dynamic noprefixroute cxgb4_iw.51 valid_lft 3475sec preferred_lft 3475sec inet6 fe80::207:43ff:fe2f:2040/64 scope link noprefixroute valid_lft forever preferred_lft forever 7: cxgb4_iw.52@cxgb4_iw: mtu 9000 qdisc noqueue state UP group default qlen 1000 link/ether 00:07:43:2f:20:40 brd ff:ff:ff:ff:ff:ff inet 172.31.52.12/24 brd 172.31.52.255 scope global dynamic noprefixroute cxgb4_iw.52 valid_lft 3475sec preferred_lft 3475sec inet6 fe80::207:43ff:fe2f:2040/64 scope link noprefixroute valid_lft forever preferred_lft forever + [22-09-24 02:31:56] grep -i fedora /etc/redhat-release + [22-09-24 02:31:56] [[ '' == \i\4\0\i\w ]] + [22-09-24 02:31:56] hostname -A + [22-09-24 02:31:56] grep rdma-qe-12 + [22-09-24 02:31:56] echo '------- start test -------' ------- start test ------- + [22-09-24 02:31:56] client + [22-09-24 02:31:56] source ./setup.sh + [22-09-24 02:31:56] client_setup + [22-09-24 02:31:56] is_cki + [22-09-24 02:31:56] typeset -l flag=true + [22-09-24 02:31:56] [[ true == \t\r\u\e ]] + [22-09-24 02:31:56] return 0 + [22-09-24 02:31:56] run_task_env_setup + [22-09-24 02:31:56] echo 'Now start to run task env_setup, please be patient ...' Now start to run task env_setup, please be patient ... + [22-09-24 02:31:56] cd ../../env_setup + [22-09-24 02:31:56] make run make[1]: Entering directory '/mnt/tests/gitlab.cee.redhat.com/kernel-qe/kernel/-/archive/master/kernel-master.zip/infiniband/env_setup' test -x runtest.sh || chmod a+x runtest.sh ./runtest.sh + trap 'exit 1' SIGHUP SIGINT SIGQUIT SIGTERM + which wget + alias + eval declare -f ++ declare -f + /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot wget /usr/bin/wget + source /usr/bin/rhts_environment.sh ++ '[' -z /mnt/testarea/tmp.jsZaRc ']' ++ '[' -z '' ']' +++ uname -i ++ ARCH=x86_64 ++ '[' -z '' ']' +++ cat /etc/redhat-release +++ sed -e 's/\(.*\)release\s\([0-9]*\).*/\1\2/; s/\s//g' ++ FAMILY=CentOSStream9 ++ '[' -h /mnt/testarea/current.log ']' ++ ln -sf /mnt/testarea/tmp.jsZaRc /mnt/testarea/current.log + '[' -f /root/rdma-qa.sh ']' + source /root/rdma-qa.sh ++ PS4='$(date +"+ [%y-%m-%d %H:%M:%S]") ' ++ [22-09-24 02:31:56] export NIC_PREF= ++ [22-09-24 02:31:56] NIC_PREF= ++ [22-09-24 02:31:56] export PERFTEST_FLAGS= ++ [22-09-24 02:31:56] PERFTEST_FLAGS= +++ [22-09-24 02:31:56] hostname -s ++ [22-09-24 02:31:56] export s_hostname=rdma-qe-12 ++ [22-09-24 02:31:56] s_hostname=rdma-qe-12 ++ [22-09-24 02:31:56] RQA_set_pyexec ++ [22-09-24 02:31:56] PYEXEC= ++ [22-09-24 02:31:56] which python ++ [22-09-24 02:31:56] export PYEXEC=python ++ [22-09-24 02:31:56] PYEXEC=python ++ [22-09-24 02:31:56] return 0 +++ [22-09-24 02:31:56] grep -i fedora /etc/redhat-release ++ [22-09-24 02:31:56] [[ -n '' ]] +++ [22-09-24 02:31:56] RQA_get_rhel_major +++ [22-09-24 02:31:56] awk -F . '{print $1}' ++++ [22-09-24 02:31:56] RQA_get_rhel_release ++++ [22-09-24 02:31:56] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [22-09-24 02:31:56] echo 9 ++ [22-09-24 02:31:56] [[ 9 -ge 8 ]] ++ [22-09-24 02:31:56] export 'PKGINSTALL=dnf install -y --setopt=strict=0 --nogpgcheck' ++ [22-09-24 02:31:56] PKGINSTALL='dnf install -y --setopt=strict=0 --nogpgcheck' ++ [22-09-24 02:31:56] export 'PKGREMOVE=dnf remove --noautoremove -y' ++ [22-09-24 02:31:56] PKGREMOVE='dnf remove --noautoremove -y' ++ [22-09-24 02:31:56] [[ rdma-qe-12 == \r\d\m\a\-\d\e\v\-\2\0 ]] ++ [22-09-24 02:31:56] export LATEST_MFT=http://www.mellanox.com/downloads/MFT/mft-4.10.0-104-x86_64-rpm.tgz ++ [22-09-24 02:31:56] LATEST_MFT=http://www.mellanox.com/downloads/MFT/mft-4.10.0-104-x86_64-rpm.tgz ++ [22-09-24 02:31:56] '[' -f /root/known-issues.sh ']' ++ [22-09-24 02:31:56] source /root/known-issues.sh ++ [22-09-24 02:31:58] RQA_load_tech_preview_modules ++ [22-09-24 02:31:58] [[ ! -e /etc/rdma/rdma.conf ]] ++ [22-09-24 02:31:58] return ++ [22-09-24 02:31:58] RQA_fix_uefi_reboot ++ [22-09-24 02:31:58] ls -l /usr/sbin/reboot ++ [22-09-24 02:31:58] grep rhts-reboot ++ [22-09-24 02:31:58] which rhts-reboot ++ [22-09-24 02:31:58] rpm -q rhts-test-env ++ [22-09-24 02:31:58] RQA_fix_ssh_ipv6_auth +++ [22-09-24 02:31:58] RQA_get_rhel_major +++ [22-09-24 02:31:58] awk -F . '{print $1}' ++++ [22-09-24 02:31:58] RQA_get_rhel_release ++++ [22-09-24 02:31:58] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [22-09-24 02:31:58] echo 9 ++ [22-09-24 02:31:58] '[' 9 -ge 7 ']' ++ [22-09-24 02:31:58] local need_restart=0 ++ [22-09-24 02:31:58] grep -E '^UseDNS yes' /etc/ssh/sshd_config ++ [22-09-24 02:31:58] grep -E '^GSSAPIStrictAcceptorCheck yes' /etc/ssh/sshd_config ++ [22-09-24 02:31:58] grep 'AddressFamily inet' /root/.ssh/config ++ [22-09-24 02:31:58] '[' 0 -eq 1 ']' ++ [22-09-24 02:31:58] RQA_fix_mlx5_ib_connected_mode +++ [22-09-24 02:31:58] RQA_get_rhel_major +++ [22-09-24 02:31:58] awk -F . '{print $1}' ++++ [22-09-24 02:31:58] RQA_get_rhel_release ++++ [22-09-24 02:31:58] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [22-09-24 02:31:58] echo 9 ++ [22-09-24 02:31:58] [[ 9 -ge 7 ]] +++ [22-09-24 02:31:58] RQA_get_rhel_minor +++ [22-09-24 02:31:58] awk -F . '{print $2}' ++++ [22-09-24 02:31:58] RQA_get_rhel_release ++++ [22-09-24 02:31:58] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [22-09-24 02:31:58] echo 9 ++ [22-09-24 02:31:58] [[ '' -ge 5 ]] +++ [22-09-24 02:31:58] RQA_get_rhel_major +++ [22-09-24 02:31:58] awk -F . '{print $1}' ++++ [22-09-24 02:31:58] RQA_get_rhel_release ++++ [22-09-24 02:31:58] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [22-09-24 02:31:58] echo 9 ++ [22-09-24 02:31:58] [[ 9 -gt 7 ]] +++ [22-09-24 02:31:58] ls /etc/sysconfig/network-scripts/ +++ [22-09-24 02:31:58] grep mlx5 +++ [22-09-24 02:31:58] grep ib +++ [22-09-24 02:31:58] grep -v '~' ++ [22-09-24 02:31:58] RQA_nfs_rhel8_temp_fix +++ [22-09-24 02:31:58] RQA_get_rhel_major +++ [22-09-24 02:31:58] awk -F . '{print $1}' ++++ [22-09-24 02:31:58] RQA_get_rhel_release ++++ [22-09-24 02:31:58] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release +++ [22-09-24 02:31:58] echo 9 ++ [22-09-24 02:31:58] (( 9 == 8 )) ++ [22-09-24 02:31:58] source /root/.bashrc ++ [22-09-24 02:31:58] [[ '' == \i\4\0\i\w ]] + [22-09-24 02:31:58] export BASHRC_FILE=/root/.bashrc + [22-09-24 02:31:58] BASHRC_FILE=/root/.bashrc + [22-09-24 02:31:58] [[ rdma-qe-12 == \r\d\m\a\-\p\e\r\f\-\0\2 ]] + [22-09-24 02:31:58] [[ rdma-qe-12 == \r\d\m\a\-\p\e\r\f\-\0\3 ]] + [22-09-24 02:31:58] export ENV_DRIVER= + [22-09-24 02:31:58] ENV_DRIVER= + [22-09-24 02:31:58] export ENV_NETWORK=iw + [22-09-24 02:31:58] ENV_NETWORK=iw ++ [22-09-24 02:31:58] hostname -s + [22-09-24 02:31:58] export host_json=./json/rdma-qe-12.json + [22-09-24 02:31:58] host_json=./json/rdma-qe-12.json + [22-09-24 02:31:58] [[ rdma-dev-13.rdma.lab.eng.rdu2.redhat.com == *\r\d\m\a\-\d\e\v\-\3\0* ]] + [22-09-24 02:31:58] [[ rdma-dev-13.rdma.lab.eng.rdu2.redhat.com == *\r\d\m\a\-\d\e\v\-\3\1* ]] + [22-09-24 02:31:58] [[ rdma-qe-12.rdma.lab.eng.rdu2.redhat.com == *\r\d\m\a\-\d\e\v\-\3\0* ]] + [22-09-24 02:31:58] [[ rdma-qe-12.rdma.lab.eng.rdu2.redhat.com == *\r\d\m\a\-\d\e\v\-\3\1* ]] + [22-09-24 02:31:58] '[' -z rdma-qe-12.rdma.lab.eng.rdu2.redhat.com -a -z rdma-dev-13.rdma.lab.eng.rdu2.redhat.com ']' + [22-09-24 02:31:58] RQA_set_servers_clients ++ [22-09-24 02:31:58] echo rdma-dev-13.rdma.lab.eng.rdu2.redhat.com ++ [22-09-24 02:31:58] awk -F . '{ print $1}' + [22-09-24 02:31:58] s_server_name=rdma-dev-13 ++ [22-09-24 02:31:58] echo rdma-qe-12.rdma.lab.eng.rdu2.redhat.com ++ [22-09-24 02:31:58] awk -F . '{ print $1}' + [22-09-24 02:31:58] s_client_name=rdma-qe-12 + [22-09-24 02:31:58] grep 'export SERVERS=' /root/.bashrc + [22-09-24 02:31:58] echo 'export SERVERS="rdma-dev-13"' + [22-09-24 02:31:58] grep 'export CLIENTS=' /root/.bashrc + [22-09-24 02:31:58] echo 'export CLIENTS="rdma-qe-12"' + [22-09-24 02:31:58] source /root/.bashrc ++ [22-09-24 02:31:58] '[' -f /etc/bashrc ']' ++ [22-09-24 02:31:58] . /etc/bashrc +++ [22-09-24 02:31:58] '[' -z Y ']' ++ [22-09-24 02:31:58] '[' vt100 = vt102 ']' ++ [22-09-24 02:31:58] alias 'grep=/bin/grep --color=tty ' ++ [22-09-24 02:31:58] alias vi=vim ++ [22-09-24 02:31:58] export EDITOR=vim ++ [22-09-24 02:31:58] EDITOR=vim ++ [22-09-24 02:31:58] RDMA_LOOKASIDE=http://download-node-02.eng.bos.redhat.com/qa/rhts/lookaside/rdma-testing +++ [22-09-24 02:31:58] which rhts-reboot +++ [22-09-24 02:31:58] alias +++ [22-09-24 02:31:58] /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot rhts-reboot +++ [22-09-24 02:31:58] eval declare -f ++++ [22-09-24 02:31:58] declare -f ++ [22-09-24 02:31:58] '[' '!' -x /usr/bin/rhts-reboot ']' ++ [22-09-24 02:31:58] export SERVERS=rdma-dev-13 ++ [22-09-24 02:31:58] SERVERS=rdma-dev-13 ++ [22-09-24 02:31:58] export CLIENTS=rdma-qe-12 ++ [22-09-24 02:31:58] CLIENTS=rdma-qe-12 + [22-09-24 02:31:58] RQA_rhts_or_dev_mode multi env + [22-09-24 02:31:58] export RUN_NUMBER=0 + [22-09-24 02:31:58] RUN_NUMBER=0 ++ [22-09-24 02:31:58] grep -i 'Job whiteboard:' /etc/motd ++ [22-09-24 02:31:58] awk -F '[' '{print $2}' ++ [22-09-24 02:31:58] sed 's/\]//g' + [22-09-24 02:31:58] merge_request= + [22-09-24 02:31:58] [[ ! -z '' ]] + [22-09-24 02:31:58] test -z 7039898 + [22-09-24 02:31:58] echo 'Variable JOBID set, we'\''re running on RHTS' Variable JOBID set, we're running on RHTS +++ [22-09-24 02:31:58] pwd ++ [22-09-24 02:31:58] basename /mnt/tests/gitlab.cee.redhat.com/kernel-qe/kernel/-/archive/master/kernel-master.zip/infiniband/env_setup + [22-09-24 02:31:58] '[' env_setup == nfsordma_sh ']' + [22-09-24 02:31:58] exec + [22-09-24 02:31:58] exec + [22-09-24 02:31:58] '[' multi == multi ']' + [22-09-24 02:31:58] echo 'Clients: rdma-qe-12' Clients: rdma-qe-12 + [22-09-24 02:31:58] echo 'Servers: rdma-dev-13' Servers: rdma-dev-13 + [22-09-24 02:31:58] RQA_system_info_for_debug + [22-09-24 02:31:58] grep -i distro /etc/motd + [22-09-24 02:31:58] tr -d ' ' DISTRO= DISTRO= + [22-09-24 02:31:58] cat /etc/redhat-release CentOS Stream release 9 + [22-09-24 02:31:58] uname -a Linux rdma-qe-12.rdma.lab.eng.rdu2.redhat.com 5.14.0-168.mr1359_220923_1526.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Sep 23 15:37:57 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux + [22-09-24 02:31:58] cat /proc/cmdline BOOT_IMAGE=(hd0,msdos1)/vmlinuz-5.14.0-168.mr1359_220923_1526.el9.x86_64 root=UUID=c34207f0-14a3-4ba4-af6a-fed8d898cbdd ro intel_idle.max_cstate=0 intremap=no_x2apic_optout processor.max_cstate=0 rd.driver.blacklist=csiostor console=tty0 rd_NO_PLYMOUTH resume=UUID=c632c698-db09-462d-a5e8-f080932a3eaa console=ttyS1,115200n81 crashkernel=1G-4G:384M,4G-16G:512M,16G-64G:1G,64G-128G:2G,128G-:4G + [22-09-24 02:31:58] rpm -q rdma-core linux-firmware rdma-core-41.0-3.el9.x86_64 linux-firmware-20220708-127.el9.noarch + [22-09-24 02:31:58] tail /sys/class/infiniband/cxgb4_0/fw_ver 1.26.6.0 + [22-09-24 02:31:58] lspci + [22-09-24 02:31:58] grep -i -e ethernet -e infiniband -e omni -e ConnectX 03:00.0 Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5717 Gigabit Ethernet PCIe (rev 10) 03:00.1 Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5717 Gigabit Ethernet PCIe (rev 10) 05:00.0 Ethernet controller: Chelsio Communications Inc T520-CR Unified Wire Ethernet Controller 05:00.1 Ethernet controller: Chelsio Communications Inc T520-CR Unified Wire Ethernet Controller 05:00.2 Ethernet controller: Chelsio Communications Inc T520-CR Unified Wire Ethernet Controller 05:00.3 Ethernet controller: Chelsio Communications Inc T520-CR Unified Wire Ethernet Controller 05:00.4 Ethernet controller: Chelsio Communications Inc T520-CR Unified Wire Ethernet Controller + [22-09-24 02:31:58] lscpu Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Address sizes: 36 bits physical, 48 bits virtual Byte Order: Little Endian CPU(s): 8 On-line CPU(s) list: 0-7 Vendor ID: GenuineIntel BIOS Vendor ID: Intel Model name: Intel(R) Xeon(R) CPU E3-1240 V2 @ 3.40GHz BIOS Model name: Intel(R) Xeon(R) CPU E3-1240 V2 @ 3.40GHz CPU family: 6 Model: 58 Thread(s) per core: 2 Core(s) per socket: 4 Socket(s): 1 Stepping: 9 CPU max MHz: 3800.0000 CPU min MHz: 1600.0000 BogoMIPS: 6784.38 Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm cpuid_fault epb pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase smep erms xsaveopt dtherm ida arat pln pts md_clear flush_l1d Virtualization: VT-x L1d cache: 128 KiB (4 instances) L1i cache: 128 KiB (4 instances) L2 cache: 1 MiB (4 instances) L3 cache: 8 MiB (1 instance) NUMA node(s): 1 NUMA node0 CPU(s): 0-7 Vulnerability Itlb multihit: KVM: Mitigation: VMX disabled Vulnerability L1tf: Mitigation; PTE Inversion; VMX conditional cache flushes, SMT vulnerable Vulnerability Mds: Mitigation; Clear CPU buffers; SMT vulnerable Vulnerability Meltdown: Mitigation; PTI Vulnerability Mmio stale data: Not affected Vulnerability Retbleed: Not affected Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization Vulnerability Spectre v2: Mitigation; Retpolines, IBPB conditional, IBRS_FW, STIBP conditional, RSB filling, PBRSB-eIBRS Not affected Vulnerability Srbds: Vulnerable: No microcode Vulnerability Tsx async abort: Not affected + [22-09-24 02:31:59] ibstat + [22-09-24 02:31:59] ibstatus Infiniband device 'cxgb4_0' port 1 status: default gid: 0007:432f:2040:0000:0000:0000:0000:0000 base lid: 0x0 sm lid: 0x0 state: 4: ACTIVE phys state: 5: LinkUp rate: 10 Gb/sec (1X FDR10) link_layer: Ethernet Infiniband device 'cxgb4_0' port 2 status: default gid: 0007:432f:2048:0000:0000:0000:0000:0000 base lid: 0x0 sm lid: 0x0 state: 1: DOWN phys state: 3: Disabled rate: 2.5 Gb/sec (1X SDR) link_layer: Ethernet + [22-09-24 02:31:59] /usr/bin/ibv_devinfo -v hca_id: cxgb4_0 transport: iWARP (1) fw_ver: 1.26.6.0 node_guid: 0207:43ff:fe2f:2040 sys_image_guid: 0207:43ff:fe2f:2040 vendor_id: 0x1425 vendor_part_id: 21505 hw_ver: 0x0 board_id: 1425.5401 phys_port_cnt: 2 max_mr_size: 0xffffffffffffffff page_size_cap: 0xffff000 max_qp: 12288 max_qp_wr: 8190 device_cap_flags: 0x00228000 MEM_WINDOW MEM_MGT_EXTENSIONS Unknown flags: 0x8000 max_sge: 4 max_sge_rd: 1 max_cq: 24576 max_cqe: 65518 max_mr: 381300 max_pd: 65536 max_qp_rd_atom: 21 max_ee_rd_atom: 0 max_res_rd_atom: 387072 max_qp_init_rd_atom: 21 max_ee_init_rd_atom: 0 atomic_cap: ATOMIC_NONE (0) max_ee: 0 max_rdd: 0 max_mw: 0 max_raw_ipv6_qp: 0 max_raw_ethy_qp: 0 max_mcast_grp: 0 max_mcast_qp_attach: 0 max_total_mcast_qp_attach: 0 max_ah: 0 max_fmr: 0 max_srq: 0 max_pkeys: 0 local_ca_ack_delay: 0 general_odp_caps: rc_odp_caps: NO SUPPORT uc_odp_caps: NO SUPPORT ud_odp_caps: NO SUPPORT xrc_odp_caps: NO SUPPORT completion_timestamp_mask not supported core clock not supported device_cap_flags_ex: 0x228000 tso_caps: max_tso: 0 rss_caps: max_rwq_indirection_tables: 0 max_rwq_indirection_table_size: 0 rx_hash_function: 0x0 rx_hash_fields_mask: 0x0 max_wq_type_rq: 0 packet_pacing_caps: qp_rate_limit_min: 0kbps qp_rate_limit_max: 0kbps tag matching not supported num_comp_vectors: 8 port: 1 state: PORT_ACTIVE (4) max_mtu: 4096 (5) active_mtu: 4096 (5) sm_lid: 0 port_lid: 0 port_lmc: 0x00 link_layer: Ethernet max_msg_sz: 0xffffffff port_cap_flags: 0x009f0000 port_cap_flags2: 0x0000 max_vl_num: invalid value (0) bad_pkey_cntr: 0x0 qkey_viol_cntr: 0x0 sm_sl: 0 pkey_tbl_len: 0 gid_tbl_len: 1 subnet_timeout: 0 init_type_reply: 0 active_width: 1X (1) active_speed: 10.0 Gbps (8) port: 2 state: PORT_DOWN (1) max_mtu: 4096 (5) active_mtu: 1024 (3) sm_lid: 0 port_lid: 0 port_lmc: 0x00 link_layer: Ethernet max_msg_sz: 0xffffffff port_cap_flags: 0x009f0000 port_cap_flags2: 0x0000 max_vl_num: invalid value (0) bad_pkey_cntr: 0x0 qkey_viol_cntr: 0x0 sm_sl: 0 pkey_tbl_len: 0 gid_tbl_len: 1 subnet_timeout: 0 init_type_reply: 0 active_width: 1X (1) active_speed: 2.5 Gbps (1) + [22-09-24 02:31:59] iblinkinfo ibwarn: [13271] get_abi_version: can't read ABI version from /sys/class/infiniband_mad/abi_version (No such file or directory): is ib_umad module loaded? ibwarn: [13271] mad_rpc_open_port: can't open UMAD port ((null):0) Failed to open (null) port 0 + [22-09-24 02:31:59] [[ '' == *\o\p\a* ]] + [22-09-24 02:31:59] ip addr show 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: lom_1: mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 34:64:a9:95:c9:3c brd ff:ff:ff:ff:ff:ff altname enp3s0f0 altname eno1 inet 10.1.217.67/24 brd 10.1.217.255 scope global dynamic noprefixroute lom_1 valid_lft 86280sec preferred_lft 86280sec inet6 2620:52:0:1d9:3664:a9ff:fe95:c93c/64 scope global dynamic noprefixroute valid_lft 2591964sec preferred_lft 604764sec inet6 fe80::3664:a9ff:fe95:c93c/64 scope link noprefixroute valid_lft forever preferred_lft forever 3: lom_2: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 34:64:a9:95:c9:3d brd ff:ff:ff:ff:ff:ff altname enp3s0f1 altname eno2 4: cxgb4_iw: mtu 9000 qdisc mq state UP group default qlen 1000 link/ether 00:07:43:2f:20:40 brd ff:ff:ff:ff:ff:ff altname enp5s0f4 altname ens2f4 inet 172.31.50.12/24 brd 172.31.50.255 scope global dynamic noprefixroute cxgb4_iw valid_lft 3472sec preferred_lft 3472sec inet6 fe80::207:43ff:fe2f:2040/64 scope link noprefixroute valid_lft forever preferred_lft forever 5: cxgb4_off: mtu 1500 qdisc mq state DOWN group default qlen 1000 link/ether 00:07:43:2f:20:48 brd ff:ff:ff:ff:ff:ff altname enp5s0f4d1 altname ens2f4d1 6: cxgb4_iw.51@cxgb4_iw: mtu 9000 qdisc noqueue state UP group default qlen 1000 link/ether 00:07:43:2f:20:40 brd ff:ff:ff:ff:ff:ff inet 172.31.51.12/24 brd 172.31.51.255 scope global dynamic noprefixroute cxgb4_iw.51 valid_lft 3472sec preferred_lft 3472sec inet6 fe80::207:43ff:fe2f:2040/64 scope link noprefixroute valid_lft forever preferred_lft forever 7: cxgb4_iw.52@cxgb4_iw: mtu 9000 qdisc noqueue state UP group default qlen 1000 link/ether 00:07:43:2f:20:40 brd ff:ff:ff:ff:ff:ff inet 172.31.52.12/24 brd 172.31.52.255 scope global dynamic noprefixroute cxgb4_iw.52 valid_lft 3472sec preferred_lft 3472sec inet6 fe80::207:43ff:fe2f:2040/64 scope link noprefixroute valid_lft forever preferred_lft forever + [22-09-24 02:31:59] PY_FILE=/root/py_parse_out.txt + [22-09-24 02:31:59] rm -f /root/py_parse_out.txt + [22-09-24 02:31:59] export result=PASS + [22-09-24 02:31:59] result=PASS + [22-09-24 02:31:59] export TEST=/kernel/infiniband/env_setup/standalone + [22-09-24 02:31:59] TEST=/kernel/infiniband/env_setup/standalone + [22-09-24 02:31:59] ip address show + [22-09-24 02:31:59] grep 'inet 172.31.' + [22-09-24 02:31:59] [[ 0 -ne 0 ]] + [22-09-24 02:31:59] install_ca_trust + [22-09-24 02:31:59] local cert_path=/etc/pki/ca-trust/source/anchors + [22-09-24 02:31:59] '[' -f /etc/pki/ca-trust/source/anchors/legacy.crt -a -f /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt ']' + [22-09-24 02:31:59] wget --no-check-certificate -P /etc/pki/ca-trust/source/anchors https://password.corp.redhat.com/legacy.crt --2022-09-24 02:31:59-- https://password.corp.redhat.com/legacy.crt Resolving password.corp.redhat.com (password.corp.redhat.com)... 10.2.111.154 Connecting to password.corp.redhat.com (password.corp.redhat.com)|10.2.111.154|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 1338 (1.3K) Saving to: ‘/etc/pki/ca-trust/source/anchors/legacy.crt’ 0K . 100% 94.5M=0s 2022-09-24 02:31:59 (94.5 MB/s) - ‘/etc/pki/ca-trust/source/anchors/legacy.crt’ saved [1338/1338] + [22-09-24 02:31:59] wget --no-check-certificate -P /etc/pki/ca-trust/source/anchors https://password.corp.redhat.com/RH-IT-Root-CA.crt --2022-09-24 02:31:59-- https://password.corp.redhat.com/RH-IT-Root-CA.crt Resolving password.corp.redhat.com (password.corp.redhat.com)... 10.2.111.154 Connecting to password.corp.redhat.com (password.corp.redhat.com)|10.2.111.154|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 1517 (1.5K) Saving to: ‘/etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt.1’ 0K . 100% 96.2M=0s 2022-09-24 02:31:59 (96.2 MB/s) - ‘/etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt.1’ saved [1517/1517] + [22-09-24 02:31:59] update-ca-trust enable + [22-09-24 02:32:00] update-ca-trust extract + [22-09-24 02:32:01] RQA_rhel8_brew_repo ++ [22-09-24 02:32:01] RQA_get_rhel_major ++ [22-09-24 02:32:01] awk -F . '{print $1}' +++ [22-09-24 02:32:01] RQA_get_rhel_release +++ [22-09-24 02:32:01] grep -o '[0-9]*\.*[0-9]*' /etc/redhat-release ++ [22-09-24 02:32:01] echo 9 + [22-09-24 02:32:01] local major=9 + [22-09-24 02:32:01] (( major == 8 )) + [22-09-24 02:32:01] pek2_lab_setup ++ [22-09-24 02:32:01] RQA_get_rdma_hostname ++ [22-09-24 02:32:01] hostname -A ++ [22-09-24 02:32:01] awk '{print $1}' ++ [22-09-24 02:32:01] grep -o 'rdma.*' ++ [22-09-24 02:32:01] awk -F . '{print $1}' + [22-09-24 02:32:01] [[ rdma-qe-12 == *\r\d\m\a\0* ]] + [22-09-24 02:32:01] common_setup ++ [22-09-24 02:32:01] RQA_get_rdma_hostname ++ [22-09-24 02:32:01] hostname -A ++ [22-09-24 02:32:01] grep -o 'rdma.*' ++ [22-09-24 02:32:01] awk '{print $1}' ++ [22-09-24 02:32:01] awk -F . '{print $1}' + [22-09-24 02:32:01] short_hostname=rdma-qe-12 + [22-09-24 02:32:01] RDMA_DRIVER= + [22-09-24 02:32:01] RDMA_NETWORK= + [22-09-24 02:32:01] '[' '!' -z ']' + [22-09-24 02:32:01] '[' '!' -z iw ']' + [22-09-24 02:32:01] RDMA_NETWORK=iw + [22-09-24 02:32:01] inv_file=json/rdma-qe-12.json + [22-09-24 02:32:01] '[' '!' -f json/rdma-qe-12.json ']' + [22-09-24 02:32:01] set -o pipefail + [22-09-24 02:32:01] python host_data_parser.py json/rdma-qe-12.json '' iw + [22-09-24 02:32:01] tee /root/py_parse_out.txt export RDMA_DRIVER="cxgb4" export RDMA_NETWORK="iw" export HCA_ID="cxgb4_0" export HCA_RATE="10 Gb/sec" export HCA_ABRV="t520" export DEVICE_ID="cxgb4_iw" export DEVICE_MAC="00:07:43:2f:20:40" export DEVICE_PORT="1" + [22-09-24 02:32:01] '[' 0 -ne 0 ']' + [22-09-24 02:32:01] source /root/py_parse_out.txt ++ [22-09-24 02:32:01] export RDMA_DRIVER=cxgb4 ++ [22-09-24 02:32:01] RDMA_DRIVER=cxgb4 ++ [22-09-24 02:32:01] export RDMA_NETWORK=iw ++ [22-09-24 02:32:01] RDMA_NETWORK=iw ++ [22-09-24 02:32:01] export HCA_ID=cxgb4_0 ++ [22-09-24 02:32:01] HCA_ID=cxgb4_0 ++ [22-09-24 02:32:01] export 'HCA_RATE=10 Gb/sec' ++ [22-09-24 02:32:01] HCA_RATE='10 Gb/sec' ++ [22-09-24 02:32:01] export HCA_ABRV=t520 ++ [22-09-24 02:32:01] HCA_ABRV=t520 ++ [22-09-24 02:32:01] export DEVICE_ID=cxgb4_iw ++ [22-09-24 02:32:01] DEVICE_ID=cxgb4_iw ++ [22-09-24 02:32:01] export DEVICE_MAC=00:07:43:2f:20:40 ++ [22-09-24 02:32:01] DEVICE_MAC=00:07:43:2f:20:40 ++ [22-09-24 02:32:01] export DEVICE_PORT=1 ++ [22-09-24 02:32:01] DEVICE_PORT=1 + [22-09-24 02:32:01] RQA_check_result -r 0 -t 'Source python parser output' + [22-09-24 02:32:01] local test_pass=0 + [22-09-24 02:32:01] local test_skip=777 + [22-09-24 02:32:01] test 4 -gt 0 + [22-09-24 02:32:01] case $1 in + [22-09-24 02:32:01] local rc=0 + [22-09-24 02:32:01] shift + [22-09-24 02:32:01] shift + [22-09-24 02:32:01] test 2 -gt 0 + [22-09-24 02:32:01] case $1 in + [22-09-24 02:32:01] local 'msg=Source python parser output' + [22-09-24 02:32:01] shift + [22-09-24 02:32:01] shift + [22-09-24 02:32:01] test 0 -gt 0 + [22-09-24 02:32:01] '[' -z 0 -o -z 'Source python parser output' ']' + [22-09-24 02:32:01] '[' -z '' ']' + [22-09-24 02:32:01] source /etc/environment + [22-09-24 02:32:01] '[' -z '' ']' + [22-09-24 02:32:01] __RQA_create_test_result_file ++ [22-09-24 02:32:01] rpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}' rdma-core + [22-09-24 02:32:01] local rdma_version=rdma-core-41.0-3.el9 ++ [22-09-24 02:32:01] echo /kernel/infiniband/env_setup/standalone ++ [22-09-24 02:32:01] sed -r 's/\/kernel\/infiniband\///;s/\/server//;s/\/client//;s/\/standalone//' + [22-09-24 02:32:01] TEST_NAME=env_setup + [22-09-24 02:32:01] [[ ! -z rdma-dev-13 ]] + [22-09-24 02:32:01] [[ ! -z rdma-qe-12 ]] + [22-09-24 02:32:01] hosts_list=rdma-dev-13/rdma-qe-12 + [22-09-24 02:32:01] local _bkrjb= ++ [22-09-24 02:32:01] grep JOBID= /etc/motd ++ [22-09-24 02:32:01] tr -d ' ' ++ [22-09-24 02:32:01] sed s/JOBID=/J:/g ++ [22-09-24 02:32:01] head -1 + [22-09-24 02:32:01] _bkrjb=J:7039898 + [22-09-24 02:32:01] [[ ! -z J:7039898 ]] + [22-09-24 02:32:01] _bkrjb=' & Beaker job J:7039898' ++ [22-09-24 02:32:01] echo results_env_setup.txt ++ [22-09-24 02:32:01] sed 's/\//-/g' + [22-09-24 02:32:01] local result_filename=results_env_setup.txt ++ [22-09-24 02:32:01] mktemp -d + [22-09-24 02:32:01] TEST_RESULT_FILE=/tmp/tmp.wzrppTPzah/results_env_setup.txt + [22-09-24 02:32:01] echo 'env_setup test results on rdma-dev-13/rdma-qe-12 & Beaker job J:7039898:' ++ [22-09-24 02:32:01] uname -r + [22-09-24 02:32:01] echo '5.14.0-168.mr1359_220923_1526.el9.x86_64, rdma-core-41.0-3.el9, cxgb4, iw, t520 & cxgb4_0' + [22-09-24 02:32:01] echo ' Result | Status | Test' + [22-09-24 02:32:01] echo ' ---------+--------+------------------------------------' + [22-09-24 02:32:01] sed -i /TEST_RESULT_FILE/d /etc/environment + [22-09-24 02:32:01] echo TEST_RESULT_FILE=/tmp/tmp.wzrppTPzah/results_env_setup.txt + [22-09-24 02:32:01] source /etc/environment ++ [22-09-24 02:32:01] TEST_RESULT_FILE=/tmp/tmp.wzrppTPzah/results_env_setup.txt + [22-09-24 02:32:01] '[' 0 -eq 0 ']' + [22-09-24 02:32:01] local test_result=PASS + [22-09-24 02:32:01] printf '%10s | %6s | %s\n' PASS 0 'Source python parser output' + [22-09-24 02:32:01] set +x --- - TEST RESULT FOR env_setup - Test: Source python parser output - Result: PASS - Return: 0 --- + [22-09-24 02:32:01] [[ cxgb4 == \r\x\e ]] + [22-09-24 02:32:01] RQA_bring_up_network iw cxgb4 + [22-09-24 02:32:01] '[' 2 -ne 2 ']' + [22-09-24 02:32:01] local _RDMA_NETWORK=iw + [22-09-24 02:32:01] local _RDMA_DRIVER=cxgb4 ++ [22-09-24 02:32:01] RQA_get_rdma_hostname ++ [22-09-24 02:32:01] hostname -A ++ [22-09-24 02:32:01] grep -o 'rdma.*' ++ [22-09-24 02:32:01] awk '{print $1}' ++ [22-09-24 02:32:01] awk -F . '{print $1}' + [22-09-24 02:32:01] local rdma_hostname=rdma-qe-12 + [22-09-24 02:32:01] [[ rdma-qe-12 == *\r\d\m\a\0* ]] + [22-09-24 02:32:01] [[ ! -z rdma-qe-12 ]] + [22-09-24 02:32:01] [[ ! -z rdma-dev-13 ]] ++ [22-09-24 02:32:01] RQA_hn_tail rdma-dev-13 ++ [22-09-24 02:32:01] echo rdma-dev-13 ++ [22-09-24 02:32:01] sed -e 's/\..*//' -e s/rdma-// + [22-09-24 02:32:01] local alias_server=iw-dev-13 ++ [22-09-24 02:32:01] cat /etc/hosts ++ [22-09-24 02:32:01] grep iw-dev-13 ++ [22-09-24 02:32:01] awk -F ' ' '{print $1}' + [22-09-24 02:32:01] IP_SERVER=172.31.50.113 ++ [22-09-24 02:32:01] RQA_hn_tail rdma-qe-12 ++ [22-09-24 02:32:01] echo rdma-qe-12 ++ [22-09-24 02:32:01] sed -e 's/\..*//' -e s/rdma-// + [22-09-24 02:32:01] local alias_client=iw-qe-12 ++ [22-09-24 02:32:01] cat /etc/hosts ++ [22-09-24 02:32:01] grep iw-qe-12 ++ [22-09-24 02:32:01] awk -F ' ' '{print $1}' + [22-09-24 02:32:01] IP_CLIENT=172.31.50.12 + [22-09-24 02:32:01] local bad=0 + [22-09-24 02:32:01] ping -c3 172.31.50.113 PING 172.31.50.113 (172.31.50.113) 56(84) bytes of data. 64 bytes from 172.31.50.113: icmp_seq=1 ttl=64 time=0.113 ms 64 bytes from 172.31.50.113: icmp_seq=2 ttl=64 time=0.063 ms 64 bytes from 172.31.50.113: icmp_seq=3 ttl=64 time=0.081 ms --- 172.31.50.113 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2045ms rtt min/avg/max/mdev = 0.063/0.085/0.113/0.020 ms + [22-09-24 02:32:03] (( bad+=0 )) + [22-09-24 02:32:03] ping -c3 172.31.50.12 PING 172.31.50.12 (172.31.50.12) 56(84) bytes of data. 64 bytes from 172.31.50.12: icmp_seq=1 ttl=64 time=0.013 ms 64 bytes from 172.31.50.12: icmp_seq=2 ttl=64 time=0.031 ms 64 bytes from 172.31.50.12: icmp_seq=3 ttl=64 time=0.045 ms --- 172.31.50.12 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2044ms rtt min/avg/max/mdev = 0.013/0.029/0.045/0.013 ms + [22-09-24 02:32:05] (( bad+=0 )) + [22-09-24 02:32:05] '[' 0 -eq 0 ']' + [22-09-24 02:32:05] return 0 + [22-09-24 02:32:05] [[ cxgb4 == \s\i\w ]] + [22-09-24 02:32:05] sleep 5 ++ [22-09-24 02:32:10] RQA_get_my_ipv4 cxgb4_iw ++ [22-09-24 02:32:10] local IPv4= ++ [22-09-24 02:32:10] '[' 1 -ne 1 ']' +++ [22-09-24 02:32:10] ip addr show dev cxgb4_iw +++ [22-09-24 02:32:10] sed '-es/^.*inet \([^ ]*\)\/.*$/\1/;t;d' ++ [22-09-24 02:32:10] IPv4=172.31.50.12 ++ [22-09-24 02:32:10] [[ 172.31.50.12 == '' ]] ++ [22-09-24 02:32:10] echo 172.31.50.12 ++ [22-09-24 02:32:10] return 0 + [22-09-24 02:32:10] RDMA_IPV4=172.31.50.12 ++ [22-09-24 02:32:10] RQA_get_my_ipv6 cxgb4_iw ++ [22-09-24 02:32:10] '[' 1 -ne 1 ']' ++ [22-09-24 02:32:10] ip addr show dev cxgb4_iw ++ [22-09-24 02:32:10] sed '-es/^.*inet6 \([^ ]*\)\/.*$/\1/;t;d' + [22-09-24 02:32:10] RDMA_IPV6=fe80::207:43ff:fe2f:2040 + [22-09-24 02:32:10] [[ cxgb4 == \s\i\w ]] + [22-09-24 02:32:10] [[ cxgb4 == \r\x\e ]] + [22-09-24 02:32:10] grep -i fedora /etc/redhat-release + [22-09-24 02:32:10] ping -c 3 172.31.50.12 PING 172.31.50.12 (172.31.50.12) 56(84) bytes of data. 64 bytes from 172.31.50.12: icmp_seq=1 ttl=64 time=0.020 ms 64 bytes from 172.31.50.12: icmp_seq=2 ttl=64 time=0.033 ms 64 bytes from 172.31.50.12: icmp_seq=3 ttl=64 time=0.029 ms --- 172.31.50.12 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2072ms rtt min/avg/max/mdev = 0.020/0.027/0.033/0.005 ms + [22-09-24 02:32:12] RQA_check_result -r 0 -t 'ping 172.31.50.12' + [22-09-24 02:32:12] local test_pass=0 + [22-09-24 02:32:12] local test_skip=777 + [22-09-24 02:32:12] test 4 -gt 0 + [22-09-24 02:32:12] case $1 in + [22-09-24 02:32:12] local rc=0 + [22-09-24 02:32:12] shift + [22-09-24 02:32:12] shift + [22-09-24 02:32:12] test 2 -gt 0 + [22-09-24 02:32:12] case $1 in + [22-09-24 02:32:12] local 'msg=ping 172.31.50.12' + [22-09-24 02:32:12] shift + [22-09-24 02:32:12] shift + [22-09-24 02:32:12] test 0 -gt 0 + [22-09-24 02:32:12] '[' -z 0 -o -z 'ping 172.31.50.12' ']' + [22-09-24 02:32:12] '[' -z /tmp/tmp.wzrppTPzah/results_env_setup.txt ']' + [22-09-24 02:32:12] '[' -z /tmp/tmp.wzrppTPzah/results_env_setup.txt ']' + [22-09-24 02:32:12] '[' 0 -eq 0 ']' + [22-09-24 02:32:12] local test_result=PASS + [22-09-24 02:32:12] printf '%10s | %6s | %s\n' PASS 0 'ping 172.31.50.12' + [22-09-24 02:32:12] set +x --- - TEST RESULT FOR env_setup - Test: ping 172.31.50.12 - Result: PASS - Return: 0 --- + [22-09-24 02:32:12] standalone + [22-09-24 02:32:12] echo -en '\n# /kernel/infiniband/env_setup variables\n' + [22-09-24 02:32:12] [[ -s /root/py_parse_out.txt ]] + [22-09-24 02:32:12] grep export /root/py_parse_out.txt + [22-09-24 02:32:12] cat + [22-09-24 02:32:12] echo 'export RDMA_IPV4="172.31.50.12"' + [22-09-24 02:32:12] echo 'export RDMA_IPV6="fe80::207:43ff:fe2f:2040"' + [22-09-24 02:32:12] source /root/.bashrc + [22-09-24 02:32:12] RQA_check_result -r 0 -t 'Source .bashrc after updates' + [22-09-24 02:32:12] local test_pass=0 + [22-09-24 02:32:12] local test_skip=777 + [22-09-24 02:32:12] test 4 -gt 0 + [22-09-24 02:32:12] case $1 in + [22-09-24 02:32:12] local rc=0 + [22-09-24 02:32:12] shift + [22-09-24 02:32:12] shift + [22-09-24 02:32:12] test 2 -gt 0 + [22-09-24 02:32:12] case $1 in + [22-09-24 02:32:12] local 'msg=Source .bashrc after updates' + [22-09-24 02:32:12] shift + [22-09-24 02:32:12] shift + [22-09-24 02:32:12] test 0 -gt 0 + [22-09-24 02:32:12] '[' -z 0 -o -z 'Source .bashrc after updates' ']' + [22-09-24 02:32:12] '[' -z /tmp/tmp.wzrppTPzah/results_env_setup.txt ']' + [22-09-24 02:32:12] '[' -z /tmp/tmp.wzrppTPzah/results_env_setup.txt ']' + [22-09-24 02:32:12] '[' 0 -eq 0 ']' + [22-09-24 02:32:12] local test_result=PASS + [22-09-24 02:32:12] printf '%10s | %6s | %s\n' PASS 0 'Source .bashrc after updates' + [22-09-24 02:32:12] set +x --- - TEST RESULT FOR env_setup - Test: Source .bashrc after updates - Result: PASS - Return: 0 --- + [22-09-24 02:32:12] RQA_overall_result + [22-09-24 02:32:12] source /root/known-issues.sh + [22-09-24 02:32:14] '[' -z /tmp/tmp.wzrppTPzah/results_env_setup.txt ']' + [22-09-24 02:32:14] '[' -z /tmp/tmp.wzrppTPzah/results_env_setup.txt ']' + [22-09-24 02:32:14] local overall_result=PASS + [22-09-24 02:32:14] echo 'Checking for failures and known issues:' + [22-09-24 02:32:14] grep FAIL /tmp/tmp.wzrppTPzah/results_env_setup.txt + [22-09-24 02:32:14] echo ' no test failures' + [22-09-24 02:32:14] echo '' + [22-09-24 02:32:14] report_result /kernel/infiniband/env_setup/standalone PASS 0 + [22-09-24 02:32:14] OUTPUTFILE=/mnt/testarea/tmp.jsZaRc + [22-09-24 02:32:14] rstrnt-report-result /kernel/infiniband/env_setup/standalone PASS 0 ** /kernel/infiniband/env_setup/standalone PASS Score:0 Uploading resultoutputfile.log .done + [22-09-24 02:32:15] rhts-submit-log -l /tmp/tmp.wzrppTPzah/results_env_setup.txt Uploading results_env_setup.txt .done + [22-09-24 02:32:15] __RQA_unset_test_result_file + [22-09-24 02:32:15] cat /tmp/tmp.wzrppTPzah/results_env_setup.txt env_setup test results on rdma-dev-13/rdma-qe-12 & Beaker job J:7039898: 5.14.0-168.mr1359_220923_1526.el9.x86_64, rdma-core-41.0-3.el9, cxgb4, iw, t520 & cxgb4_0 Result | Status | Test ---------+--------+------------------------------------ PASS | 0 | Source python parser output PASS | 0 | ping 172.31.50.12 PASS | 0 | Source .bashrc after updates Checking for failures and known issues: no test failures + [22-09-24 02:32:15] export TEST_RESULT_FILE= + [22-09-24 02:32:15] TEST_RESULT_FILE= + [22-09-24 02:32:15] unset TEST_RESULT_FILE + [22-09-24 02:32:15] sed -i /TEST_RESULT_FILE/d /etc/environment + [22-09-24 02:32:15] echo 'TEST_RESULT_FILE=""' + [22-09-24 02:32:15] source /etc/environment ++ [22-09-24 02:32:15] TEST_RESULT_FILE= + [22-09-24 02:32:15] return 0 + [22-09-24 02:32:15] hostname -A + [22-09-24 02:32:15] grep rdma-qe-12 + [22-09-24 02:32:15] echo '------- client start test -------' ------- client start test ------- + [22-09-24 02:32:15] export result=PASS + [22-09-24 02:32:15] result=PASS + [22-09-24 02:32:15] export TEST=/kernel/infiniband/env_setup/standalone/client + [22-09-24 02:32:15] TEST=/kernel/infiniband/env_setup/standalone/client + [22-09-24 02:32:15] client + [22-09-24 02:32:15] [[ cxgb4 == \s\i\w ]] + [22-09-24 02:32:15] rhts_sync_set -s ES_CLIENT_READY_-iw-0 /usr/bin/rhts_sync_set -s ES_CLIENT_READY_-iw-0 -- State set = 10_ES_CLIENT_READY_-iw-0 + [22-09-24 02:32:15] rhts_sync_block -s ES_SERVER_READY_-iw-0 rdma-dev-13 /usr/bin/rhts_sync_block -s ES_SERVER_READY_-iw-0 rdma-dev-13 -- Blocking state(s) = 10_ES_SERVER_READY_-iw-0 Failed to connect to remote server rdma-dev-13: Connection refused Retrying rstrnt-sync-block for state 10_ES_SERVER_READY_-iw-0 + [22-09-24 02:33:15] rhts_sync_block -s ES_SERVER_DONE_-iw-0 rdma-dev-13 /usr/bin/rhts_sync_block -s ES_SERVER_DONE_-iw-0 rdma-dev-13 -- Blocking state(s) = 10_ES_SERVER_DONE_-iw-0 ++ [22-09-24 02:33:15] echo rdma-dev-13 ++ [22-09-24 02:33:15] awk -F . '{print $1}' + [22-09-24 02:33:15] srv_ssh=rdma-dev-13 ++ [22-09-24 02:33:15] ssh rdma-dev-13 echo '${RDMA_DRIVER}' + [22-09-24 02:33:16] SERVER_DRIVER=cxgb4 ++ [22-09-24 02:33:16] ssh rdma-dev-13 echo '${RDMA_NETWORK}' + [22-09-24 02:33:16] SERVER_NETWORK=iw ++ [22-09-24 02:33:16] ssh rdma-dev-13 echo '${HCA_ID}' + [22-09-24 02:33:16] SERVER_HCA_ID=cxgb4_0 ++ [22-09-24 02:33:16] ssh rdma-dev-13 echo '${HCA_ABRV}' + [22-09-24 02:33:17] SERVER_HCA_ABRV=t62100 ++ [22-09-24 02:33:17] ssh rdma-dev-13 echo '${DEVICE_ID}' + [22-09-24 02:33:17] SERVER_DEVICE_ID=cxgb4_iw ++ [22-09-24 02:33:17] ssh rdma-dev-13 echo '${DEVICE_MAC}' + [22-09-24 02:33:17] SERVER_DEVICE_MAC=00:07:43:3b:f4:d0 ++ [22-09-24 02:33:17] ssh rdma-dev-13 echo '${DEVICE_PORT}' + [22-09-24 02:33:17] SERVER_DEVICE_PORT=1 ++ [22-09-24 02:33:17] ssh rdma-dev-13 echo '${RDMA_IPV4}' + [22-09-24 02:33:17] SERVER_IPV4=172.31.50.113 ++ [22-09-24 02:33:17] ssh rdma-dev-13 echo '${RDMA_IPV6}' + [22-09-24 02:33:18] SERVER_IPV6=fe80::207:43ff:fe3b:f4d0 + [22-09-24 02:33:18] '[' -z cxgb4 ']' + [22-09-24 02:33:18] '[' -z iw ']' + [22-09-24 02:33:18] '[' -z cxgb4_0 ']' + [22-09-24 02:33:18] '[' -z t62100 ']' + [22-09-24 02:33:18] '[' -z cxgb4_iw ']' + [22-09-24 02:33:18] '[' -z 00:07:43:3b:f4:d0 ']' + [22-09-24 02:33:18] '[' -z 1 ']' + [22-09-24 02:33:18] '[' -z 172.31.50.113 ']' + [22-09-24 02:33:18] '[' -z fe80::207:43ff:fe3b:f4d0 ']' + [22-09-24 02:33:18] echo 'export CLIENT_DRIVER="cxgb4"' + [22-09-24 02:33:18] echo 'export CLIENT_NETWORK="iw"' + [22-09-24 02:33:18] echo 'export CLIENT_HCA_ID="cxgb4_0"' + [22-09-24 02:33:18] echo 'export CLIENT_HCA_ABRV="t520"' + [22-09-24 02:33:18] echo 'export CLIENT_DEVICE_ID="cxgb4_iw"' + [22-09-24 02:33:18] echo 'export CLIENT_DEVICE_MAC="00:07:43:2f:20:40"' + [22-09-24 02:33:18] echo 'export CLIENT_DEVICE_PORT="1"' + [22-09-24 02:33:18] echo 'export CLIENT_IPV4="172.31.50.12"' + [22-09-24 02:33:18] echo 'export CLIENT_IPV6="fe80::207:43ff:fe2f:2040"' + [22-09-24 02:33:18] echo 'export SERVER_DRIVER="cxgb4"' + [22-09-24 02:33:18] echo 'export SERVER_NETWORK="iw"' + [22-09-24 02:33:18] echo 'export SERVER_HCA_ID="cxgb4_0"' + [22-09-24 02:33:18] echo 'export SERVER_HCA_ABRV="t62100"' + [22-09-24 02:33:18] echo 'export SERVER_DEVICE_ID="cxgb4_iw"' + [22-09-24 02:33:18] echo 'export SERVER_DEVICE_MAC="00:07:43:3b:f4:d0"' + [22-09-24 02:33:18] echo 'export SERVER_DEVICE_PORT="1"' + [22-09-24 02:33:18] echo 'export SERVER_IPV4="172.31.50.113"' + [22-09-24 02:33:18] echo 'export SERVER_IPV6="fe80::207:43ff:fe3b:f4d0"' + [22-09-24 02:33:18] echo 'export REMOTE_HOST="rdma-dev-13"' + [22-09-24 02:33:18] source /root/.bashrc + [22-09-24 02:33:18] RQA_check_result -r 0 -t 'source .bashrc in client function' + [22-09-24 02:33:18] local test_pass=0 + [22-09-24 02:33:18] local test_skip=777 + [22-09-24 02:33:18] test 4 -gt 0 + [22-09-24 02:33:18] case $1 in + [22-09-24 02:33:18] local rc=0 + [22-09-24 02:33:18] shift + [22-09-24 02:33:18] shift + [22-09-24 02:33:18] test 2 -gt 0 + [22-09-24 02:33:18] case $1 in + [22-09-24 02:33:18] local 'msg=source .bashrc in client function' + [22-09-24 02:33:18] shift + [22-09-24 02:33:18] shift + [22-09-24 02:33:18] test 0 -gt 0 + [22-09-24 02:33:18] '[' -z 0 -o -z 'source .bashrc in client function' ']' + [22-09-24 02:33:18] '[' -z '' ']' + [22-09-24 02:33:18] source /etc/environment ++ [22-09-24 02:33:18] TEST_RESULT_FILE= + [22-09-24 02:33:18] '[' -z '' ']' + [22-09-24 02:33:18] __RQA_create_test_result_file ++ [22-09-24 02:33:18] rpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}' rdma-core + [22-09-24 02:33:18] local rdma_version=rdma-core-41.0-3.el9 ++ [22-09-24 02:33:18] echo /kernel/infiniband/env_setup/standalone/client ++ [22-09-24 02:33:18] sed -r 's/\/kernel\/infiniband\///;s/\/server//;s/\/client//;s/\/standalone//' + [22-09-24 02:33:18] TEST_NAME=env_setup + [22-09-24 02:33:18] [[ ! -z rdma-dev-13 ]] + [22-09-24 02:33:18] [[ ! -z rdma-qe-12 ]] + [22-09-24 02:33:18] hosts_list=rdma-dev-13/rdma-qe-12 + [22-09-24 02:33:18] local _bkrjb= ++ [22-09-24 02:33:18] grep JOBID= /etc/motd ++ [22-09-24 02:33:18] tr -d ' ' ++ [22-09-24 02:33:18] sed s/JOBID=/J:/g ++ [22-09-24 02:33:18] head -1 + [22-09-24 02:33:18] _bkrjb=J:7039898 + [22-09-24 02:33:18] [[ ! -z J:7039898 ]] + [22-09-24 02:33:18] _bkrjb=' & Beaker job J:7039898' ++ [22-09-24 02:33:18] echo results_env_setup.txt ++ [22-09-24 02:33:18] sed 's/\//-/g' + [22-09-24 02:33:18] local result_filename=results_env_setup.txt ++ [22-09-24 02:33:18] mktemp -d + [22-09-24 02:33:18] TEST_RESULT_FILE=/tmp/tmp.aP2Emy2ktt/results_env_setup.txt + [22-09-24 02:33:18] echo 'env_setup test results on rdma-dev-13/rdma-qe-12 & Beaker job J:7039898:' ++ [22-09-24 02:33:18] uname -r + [22-09-24 02:33:18] echo '5.14.0-168.mr1359_220923_1526.el9.x86_64, rdma-core-41.0-3.el9, cxgb4, iw, t520 & cxgb4_0' + [22-09-24 02:33:18] echo ' Result | Status | Test' + [22-09-24 02:33:18] echo ' ---------+--------+------------------------------------' + [22-09-24 02:33:18] sed -i /TEST_RESULT_FILE/d /etc/environment + [22-09-24 02:33:18] echo TEST_RESULT_FILE=/tmp/tmp.aP2Emy2ktt/results_env_setup.txt + [22-09-24 02:33:18] source /etc/environment ++ [22-09-24 02:33:18] TEST_RESULT_FILE=/tmp/tmp.aP2Emy2ktt/results_env_setup.txt + [22-09-24 02:33:18] '[' 0 -eq 0 ']' + [22-09-24 02:33:18] local test_result=PASS + [22-09-24 02:33:18] printf '%10s | %6s | %s\n' PASS 0 'source .bashrc in client function' + [22-09-24 02:33:18] set +x --- - TEST RESULT FOR env_setup - Test: source .bashrc in client function - Result: PASS - Return: 0 --- + [22-09-24 02:33:18] echo '--- client finishes.' --- client finishes. + [22-09-24 02:33:18] rhts_sync_set -s ES_CLIENT_DONE_-iw-0 /usr/bin/rhts_sync_set -s ES_CLIENT_DONE_-iw-0 -- State set = 10_ES_CLIENT_DONE_-iw-0 + [22-09-24 02:33:18] RQA_overall_result + [22-09-24 02:33:18] source /root/known-issues.sh + [22-09-24 02:33:20] '[' -z /tmp/tmp.aP2Emy2ktt/results_env_setup.txt ']' + [22-09-24 02:33:20] '[' -z /tmp/tmp.aP2Emy2ktt/results_env_setup.txt ']' + [22-09-24 02:33:20] local overall_result=PASS + [22-09-24 02:33:20] echo 'Checking for failures and known issues:' + [22-09-24 02:33:20] grep FAIL /tmp/tmp.aP2Emy2ktt/results_env_setup.txt + [22-09-24 02:33:20] echo ' no test failures' + [22-09-24 02:33:20] echo '' + [22-09-24 02:33:20] report_result /kernel/infiniband/env_setup/standalone/client PASS 0 + [22-09-24 02:33:20] OUTPUTFILE=/mnt/testarea/tmp.jsZaRc + [22-09-24 02:33:20] rstrnt-report-result /kernel/infiniband/env_setup/standalone/client PASS 0 ** /kernel/infiniband/env_setup/standalone/client PASS Score:0 Uploading resultoutputfile.log