Found 307 jobs Missing teuthology log /home/teuthworker/archive/yuriw-2023-08-15_18:58:56-rados-wip-yuri3-testing-2023-08-15-0955-distro-default-smithi/7369396/teuthology.log Found 16 distinct failure reasons Dead: None 63 jobs Failure: Command failed on smithi204 with status 1: 'sudo yum install -y kernel' 23 jobs: ['7369422', '7369335', '7369448', '7369385', '7369379', '7369449', '7369345', '7369392', '7369346', '7369459', '7369343', '7369390', '7369466', '7369415', '7369420', '7369464', '7369337', '7369380', '7369401', '7369403', '7369405', '7369391', '7369439'] suites intersection: [] suites union: ['1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-inject-leak/none', '2-inject-leak/osd', '2-recovery-overrides/{default}', '2-recovery-overrides/{more-active-recovery}', '2-recovery-overrides/{more-async-recovery}', '3-scrub-overrides/{default}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'backoff/peering', 'backoff/peering_and_degraded', 'centos_latest}', 'ceph', 'clusters/3-mons', 'clusters/{fixed-2', 'clusters/{fixed-4', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'd-thrash/default/{default', 'd-thrash/none', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/mon-delay', 'msgr-failures/none', 'msgr-failures/osd-delay', 'msgr-failures/osd-dispatch-delay', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/monthrash/{ceph', 'rados/singleton-bluestore/{all/cephtool', 'rados/singleton-nomsgr/{all/balancer', 'rados/singleton-nomsgr/{all/ceph-kvstore-tool', 'rados/singleton-nomsgr/{all/crushdiff', 'rados/singleton-nomsgr/{all/health-warnings', 'rados/singleton-nomsgr/{all/librados_hello_world', 'rados/singleton/{all/pg-autoscaler', 'rados/singleton/{all/pg-autoscaler-progress-off', 'rados/singleton/{all/resolve_stuck_peering', 'rados/singleton/{all/test-crash', 'rados/thrash-erasure-code-shec/{ceph', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'rados/valgrind-leaks/{1-start', 'rados/verify/{centos_latest', 'recovery-overrides/{default}', 'supported-random-distro$/{centos_latest}', 'supported-random-distro$/{centos_latest}}', 'tasks/mon_recovery', 'tasks/rados_api_tests', 'tasks/rados_cls_all', 'thrashers/careful', 'thrashers/default', 'thrashers/many', 'thrashers/mapgap', 'thrashers/morepggrow', 'thrashers/none', 'thrashers/sync-many', 'thrashosds-health', 'thrashosds-health}', 'validater/lockdep}', 'validater/valgrind}', 'workloads/cache-pool-snaps-readproxy}', 'workloads/cache-pool-snaps}', 'workloads/cache-snaps}', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}', 'workloads/pool-create-delete}', 'workloads/rados_api_tests}', 'workloads/rados_mon_osdmap_prune}'] Failure: Command crashed: 'sudo TESTDIR=/home/ubuntu/cephtest bash -c ceph_test_lazy_omap_stats' 1 jobs: ['7369175'] suites: ['mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/lazy_omap_stats_output', 'supported-random-distro$/{ubuntu_latest}}'] Failure: SELinux denials found on ubuntu@smithi118.front.sepia.ceph.com: ['type=AVC msg=audit(1692159473.142:21140): avc: denied { ioctl } for pid=114246 comm="iptables" path="/var/lib/containers/storage/overlay/23c4a40d0efd4d0cd88683a8c30f3ed1ed9530d7f5d8cadf18d15a0f24657daa/merged" dev="overlay" ino=2754165 scontext=system_u:system_r:iptables_t:s0 tcontext=system_u:object_r:container_file_t:s0:c1022,c1023 tclass=dir permissive=1'] 1 jobs: ['7369450'] suites: ['agent/on', 'mon_election/classic', 'rados/cephadm/workunits/{0-distro/rhel_8.6_container_tools_rhel8', 'task/test_cephadm}'] Timeout 3h running clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh 1 jobs: ['7369400'] suites: ['0-start', '1-tasks', 'mon_election/connectivity', 'rados/upgrade/parallel/{0-random-distro$/{ubuntu_20.04}', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'test_rbd_api', 'test_rbd_python}}', 'upgrade-sequence', 'workload/{ec-rados-default'] Failure: ['error: package katello-ca-consumer-satellite.front.sepia.ceph.com-1.0-1.noarch is not installed'] 1 jobs: ['7369342'] suites: ['0-nvme-loop', '1-start', '2-ops/rm-zap-add}', 'rados/cephadm/osds/{0-distro/rhel_8.6_container_tools_rhel8'] Failure: "2023-08-16T02:56:09.559181+0000 osd.6 (osd.6) 4 : cluster [WRN] osd.6 PgScrubber: 15.8 timeout on reserving replicas (since 2023-08-16T02:56:04:533663+0000)" in cluster log 2 jobs: ['7369280', '7369338'] suites intersection: ['clusters/{fixed-2', 'msgr-failures/few', 'openstack}', 'rados'] suites union: ['ceph', 'clusters/{fixed-2', 'd-thrash/none', 'fast/normal', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'rados', 'rados/thrash-erasure-code/{ceph', 'rados/verify/{centos_latest', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/rados_api_tests', 'thrashers/careful', 'thrashosds-health', 'validater/valgrind}', 'workloads/ec-radosbench}'] Failure: Command failed (workunit test cephadm/test_dashboard_e2e.sh) on smithi119 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b5259484dbc61e8573f27e692560c6c107b9d4c1 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_dashboard_e2e.sh' 2 jobs: ['7369475', '7369319'] suites intersection: ['debug/mgr', 'rados/dashboard/{0-single-container-host', 'tasks/e2e}'] suites union: ['debug/mgr', 'mon_election/classic', 'mon_election/connectivity', 'rados/dashboard/{0-single-container-host', 'random-objectstore$/{bluestore-bitmap}', 'random-objectstore$/{bluestore-comp-lz4}', 'tasks/e2e}'] Failure: Test failure: test_cluster_info (tasks.cephfs.test_nfs.TestNFS) 1 jobs: ['7369221'] suites: ['agent/on', 'mon_election/classic', 'rados/cephadm/workunits/{0-distro/ubuntu_20.04', 'task/test_nfs}'] Failure: smithi102.front.sepia.ceph.com: _ansible_no_log: false changed: false invocation: module_args: allow_downgrade: false allowerasing: false autoremove: false bugfix: false conf_file: null disable_excludes: null disable_gpg_check: false disable_plugin: [] disablerepo: [] download_dir: null download_only: false enable_plugin: [] enablerepo: [] exclude: [] install_repoquery: true install_weak_deps: true installroot: / list: null lock_timeout: 30 name: - krb5-workstation releasever: null security: false skip_broken: false state: present update_cache: false update_only: false validate_certs: true msg: 'Failed to download metadata for repo ''baseos'': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried' rc: 1 results: []smithi142.front.sepia.ceph.com: _ansible_no_log: false changed: false invocation: module_args: allow_downgrade: false allowerasing: false autoremove: false bugfix: false conf_file: null disable_excludes: null disable_gpg_check: false disable_plugin: [] disablerepo: [] download_dir: null download_only: false enable_plugin: [] enablerepo: [] exclude: [] install_repoquery: true install_weak_deps: true installroot: / list: null lock_timeout: 30 name: - krb5-workstation releasever: null security: false skip_broken: false state: present update_cache: false update_only: false validate_certs: true msg: 'Failed to download metadata for repo ''baseos'': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried' rc: 1 results: [] 2 jobs: ['7369327', '7369328'] suites intersection: ['mon_election/connectivity', 'objectstore/bluestore-comp-snappy', 'rados', 'supported-random-distro$/{centos_latest}'] suites union: ['cluster/{12-osds', 'mon_election/connectivity', 'msgr-failures/many', 'msgr-failures/osd-delay', 'msgr/async', 'no_pools', 'objectstore/bluestore-comp-snappy', 'openstack}', 'rados', 'rados/multimon/{clusters/3', 'rados/thrash-erasure-code-big/{ceph', 'recovery-overrides/{more-async-recovery}', 'supported-random-distro$/{centos_latest}', 'tasks/mon_clock_no_skews}', 'thrashers/pggrow', 'thrashosds-health', 'workloads/ec-rados-plugin=jerasure-k=4-m=2}'] Failure: ['/dev/vg_nvme: already exists in filesystem', "Creating volume group 'vg_nvme' failed", "Run `vgcreate --help' for more information."] 2 jobs: ['7369285', '7369275'] suites intersection: ['mon_election/connectivity'] suites union: ['mon_election/connectivity', 'objectstore/bluestore-basic-min-osd-mem-target', 'openstack', 'rados', 'rados/perf/{ceph', 'rados/singleton-nomsgr/{all/crushdiff', 'scheduler/dmclock_1Shard_16Threads', 'settings/optimized', 'supported-random-distro$/{centos_8}}', 'ubuntu_latest', 'workloads/radosbench_4K_seq_read}'] Dead: b'2023-08-16T15:02:27.389 DEBUG:teuthology.exit:Finished running handlers' 3 jobs: ['7369344', '7369398', '7369244'] suites intersection: ['rados'] suites union: ['mon_election/classic', 'mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/admin_socket_output', 'rados/singleton-nomsgr/{all/msgr', 'supported-random-distro$/{rhel_8}}', 'supported-random-distro$/{ubuntu_latest}}'] Dead: b"TypeError: argument of type 'NoneType' is not iterable" 1 jobs: ['7369418'] suites: ['0-nvme-loop', 'agent/on', 'fixed-2', 'mon_election/connectivity', 'rados/cephadm/smoke/{0-distro/rhel_8.6_container_tools_3.0', 'start}'] Failure: "2023-08-16T02:44:03.765441+0000 mon.a (mon.0) 683 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED)" in cluster log 2 jobs: ['7369208', '7369458'] suites intersection: ['mon_election/classic', 'rados'] suites union: ['ceph', 'clusters/{fixed-2', 'd-thrash/none', 'mon_election/classic', 'msgr-failures/few', 'msgr-failures/many', 'msgr/async', 'msgr/async-v1only', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'rados', 'rados/singleton-bluestore/{all/cephtool', 'rados/verify/{centos_latest', 'supported-random-distro$/{ubuntu_latest}}', 'tasks/rados_cls_all', 'validater/valgrind}'] Failure: Command failed (workunit test scrub/osd-recovery-scrub.sh) on smithi150 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b5259484dbc61e8573f27e692560c6c107b9d4c1 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/scrub/osd-recovery-scrub.sh' 1 jobs: ['7369212'] suites: ['rados/standalone/{supported-random-distro$/{rhel_8}', 'workloads/scrub}'] Failure: Command failed (workunit test mon/test_noautoscale_flag.sh) on smithi174 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b5259484dbc61e8573f27e692560c6c107b9d4c1 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/mon/test_noautoscale_flag.sh' 1 jobs: ['7369409'] suites: ['mon_election/classic', 'msgr-failures/many', 'msgr/async-v1only', 'objectstore/bluestore-comp-lz4', 'rados', 'rados/singleton/{all/test-noautoscale-flag', 'supported-random-distro$/{rhel_8}}']