Found 10 jobs Found 6 distinct failure reasons Dead: b'2023-12-09T12:59:22.384 INFO:journalctl@ceph.mon.c.smithi012.stdout:Dec 09 12:59:21 smithi012 bash[108884]: cluster 2023-12-09T12:59:20.404732+0000 mgr.y (mgr.34107) 22991 : cluster 0 pgmap v23606: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail' 4 jobs: ['7484551', '7484550', '7484554', '7484552'] suites intersection: ['0-roles', '1-start', '3-stress-tasks/{radosbench', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'snaps-few-objects}'] suites union: ['0-roles', '1-start', '2-first-half-tasks/radosbench', '2-first-half-tasks/rbd-cls', '2-first-half-tasks/readwrite', '2-first-half-tasks/snaps-few-objects', '3-stress-tasks/{radosbench', '4-second-half-tasks/radosbench', '4-second-half-tasks/rbd-import-export', 'mon_election/classic}', 'mon_election/connectivity}', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'snaps-few-objects}', 'upgrade:quincy-x/stress-split/{0-distro/centos_8.stream_container_tools', 'upgrade:quincy-x/stress-split/{0-distro/centos_8.stream_container_tools_crun', 'upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8', 'upgrade:quincy-x/stress-split/{0-distro/ubuntu_20.04'] Dead: b'2023-12-09T12:59:22.279 INFO:journalctl@ceph.mon.c.smithi167.stdout:Dec 09 12:59:21 smithi167 bash[105804]: audit 2023-12-09T12:59:21.309222+0000 mon.a (mon.0) 13247 : audit 0 from=\'mgr.34101 172.21.15.167:0/2576690381\' entity=\'mgr.y\' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch' 1 jobs: ['7484559'] suites: ['0-roles', '1-start', '2-first-half-tasks/readwrite', '3-stress-tasks/{radosbench', '4-second-half-tasks/rbd-import-export', 'mon_election/classic}', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'snaps-few-objects}', 'upgrade:quincy-x/stress-split/{0-distro/ubuntu_20.04'] Dead: b'2023-12-09T12:59:21.848 INFO:journalctl@ceph.mon.b.smithi175.stdout:Dec 09 12:59:21 smithi175 ceph-mon[178837]: from=\'mgr.25024 172.21.15.136:0/2378255861\' entity=\'mgr.y\' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch' 1 jobs: ['7484558'] suites: ['0-start', '1-tasks', 'mon_election/classic', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'test_rbd_api', 'test_rbd_python}}', 'upgrade-sequence', 'upgrade:quincy-x/parallel/{0-random-distro$/{centos_8.stream_container_tools_crun}', 'workload/{ec-rados-default'] Failure: "1702084618.591141 mon.a (mon.0) 63 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED)" in cluster log 1 jobs: ['7484557'] suites: ['-', '1-ceph-install/quincy', '2', 'objectstore/filestore-xfs', 'start}', 'ubuntu_20.04}', 'upgrade', 'upgrade:quincy-x/filestore-remove-check/{0-cluster/{openstack'] Dead: b'2023-12-09T12:59:24.155 INFO:journalctl@ceph.mon.b.smithi120.stdout:Dec 09 12:59:23 smithi120 ceph-mon[182411]: pgmap v23194: 105 pgs: 105 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail' 2 jobs: ['7484553', '7484556'] suites intersection: [] suites union: ['0-roles', '0-start', '1-start', '1-tasks', '2-first-half-tasks/rbd_api', '3-stress-tasks/{radosbench', '4-second-half-tasks/radosbench', 'mon_election/connectivity', 'mon_election/connectivity}', 'rados_api', 'rados_loadgenbig', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'rbd_import_export', 'readwrite', 'snaps-few-objects}', 'test_rbd_api', 'test_rbd_python}}', 'upgrade-sequence', 'upgrade:quincy-x/parallel/{0-random-distro$/{centos_8.stream_container_tools}', 'upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8', 'workload/{ec-rados-default'] Failure: smithi029.front.sepia.ceph.com: _ansible_no_log: false changed: false invocation: module_args: allow_downgrade: false allowerasing: false autoremove: false bugfix: false conf_file: null disable_excludes: null disable_gpg_check: false disable_plugin: [] disablerepo: [] download_dir: null download_only: false enable_plugin: [] enablerepo: - epel exclude: [] install_repoquery: true install_weak_deps: true installroot: / list: null lock_timeout: 30 name: - nagios-common - nrpe - nagios-plugins - nagios-plugins-load releasever: null security: false skip_broken: false state: latest update_cache: false update_only: false validate_certs: true msg: 'Failed to download packages: Status code: 503 for https://mirrors.fedoraproject.org/metalink?repo=epel-8&arch=x86_64&infra=$infra&content=$contentdir (IP: 8.43.85.67)' results: [] 1 jobs: ['7484555'] suites: ['0-roles', '1-start', '2-first-half-tasks/rbd-import-export', '3-stress-tasks/{radosbench', '4-second-half-tasks/rbd-import-export', 'mon_election/classic}', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'snaps-few-objects}', 'upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0']