Found 53 jobs
Missing teuthology log /home/teuthworker/archive/adking-2023-06-29_13:57:59-orch:cephadm-wip-adk3-testing-2023-06-27-0854-distro-default-smithi/7321531/teuthology.log
Found 13 distinct failure reasons
Dead: None
9 jobs: ['7321531', '7321527', '7321550', '7321519', '7321535', '7321523', '7321538', '7321544', '7321556']

Failure: "/var/log/ceph/bdadce5a-16b5-11ee-9b2e-001a4aab830c/ceph-mon.smithi031.log:2023-06-29T19:52:28.663+0000 7fec50310700 10 mon.smithi031@0(leader).log v214  logging 2023-06-29T19:52:28.599368+0000 mgr.smithi031.rqovpj (mgr.14229) 153 : cephadm [ERR] Failed while placing nfs.foo.0.0.smithi031.xpmpcw on smithi031: grace tool failed: rados_pool_create: -1" in cluster log
3 jobs: ['7321549', '7321555', '7321551']
suites intersection: ['0-nvme-loop', '1-start', '3-final}']
suites union: ['0-nvme-loop', '1-start', '2-services/nfs', '2-services/nfs-ingress', '2-services/nfs-keepalive-only', '3-final}', 'orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_rhel8', 'orch:cephadm/smoke-roleless/{0-distro/ubuntu_20.04']

Failure: "/var/log/ceph/b52c057c-16b4-11ee-9b2e-001a4aab830c/ceph-mon.a.log:2023-06-29T19:49:59.999+0000 7f1b3e87f700  0 log_channel(cluster) log [WRN] : Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled" in cluster log
14 jobs: ['7321510', '7321530', '7321545', '7321534', '7321520', '7321505', '7321554', '7321513', '7321533', '7321547', '7321553', '7321552', '7321506', '7321539']
suites intersection: []
suites union: ['0-nvme-loop', '1-bootstrap/16.2.4', '1-ranks/1', '1-ranks/2', '1-start', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client', '2-nfs', '2-ops/rm-zap-add}', '2-ops/rm-zap-wait}', '2-repo_digest/repo_digest', '2-thrash', '3-inline/no', '3-inline/yes', '3-tasks/rados_api_tests', '3-tasks/snaps-few-objects', '3-upgrade-mgr-staggered', '3-upgrade-with-workload', '3-upgrade/simple', '4-config-upgrade/{fail_fs}', '4-final}', '4-verify}', '4-wait', '5-upgrade-ls', '5-upgrade-with-workload', '6-verify}}', 'agent/off', 'agent/on', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/no', 'fail_fs/yes', 'fixed-2', 'ignorelist_wrongly_marked_down', 'mds', 'mode/root', 'mon', 'mon_election/classic', 'mon_election/connectivity', 'mon_election/connectivity}', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools', 'orch:cephadm/osds/{0-distro/rhel_8.6_container_tools_3.0', 'orch:cephadm/osds/{0-distro/ubuntu_20.04', 'orch:cephadm/smoke-small/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/smoke/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/smoke/{0-distro/ubuntu_20.04', 'orch:cephadm/thrash/{0-distro/centos_8.stream_container_tools', 'orch:cephadm/thrash/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04', 'orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools', 'orch:cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/workunits/{0-distro/ubuntu_20.04', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'root}', 'start', 'start}', 'syntax}', 'task/test_iscsi_container/{centos_8.stream_container_tools', 'task/test_orch_cli_mon}', 'tasks/rados_api_tests}', 'tasks/{0-from/pacific', 'tasks/{0-from/v16.2.4', 'test_iscsi_container}}']

Failure: "/var/log/ceph/246ee1fa-16b7-11ee-9b2e-001a4aab830c/ceph-mon.smithi151.log:2023-06-29T20:05:58.922+0000 7f93d3411700 10 mon.smithi151@0(leader).log v325  logging 2023-06-29T20:05:58.626998+0000 mgr.smithi156.uzjovt (mgr.24373) 98 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place <MONSpec for service_name=mon>: No matching hosts for label _admin" in cluster log
1 jobs: ['7321528']
suites: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/no', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/v16.2.4']

Failure: "/var/log/ceph/b33c43e2-16b6-11ee-9b2e-001a4aab830c/ceph-mon.a.log:2023-06-29T20:03:45.652+0000 7fbb00999700  0 log_channel(cluster) log [WRN] : Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log
3 jobs: ['7321548', '7321515', '7321517']
suites intersection: []
suites union: ['1-ranks/2', '1-start', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '2-repo_digest/repo_digest', '2-thrash', '3-inline/yes', '3-tasks/snaps-few-objects', '3-upgrade-mgr-staggered', '3-upgrade/simple', '4-config-upgrade/{fail_fs}', '4-verify}', '4-wait', '5-upgrade-ls', '5-upgrade-with-workload', '6-verify}}', 'agent/off', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/yes', 'fixed-2', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon_election/classic}', 'msgr/async', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch:cephadm/thrash/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/upgrade/{1-start-distro/1-start-centos_8.stream_container-tools', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'root}', 'syntax}', 'tasks/{0-from/pacific']

Failure: "/var/log/ceph/67d33938-16b6-11ee-9b2e-001a4aab830c/ceph-mon.a.log:2023-06-29T20:04:52.004+0000 7f2a10cc2700 10 mon.a@0(leader).log v535  logging 2023-06-29T20:04:51.413410+0000 mgr.x (mgr.14604) 27 : cephadm [ERR] Non-zero return from ['ceph', '-k', '/var/lib/ceph/mgr/ceph-x/keyring', '-n', 'mgr.x', 'tell', 'mgr.x', 'rotate-key', '-i', '-']: 2023-06-29T20:04:51.406+0000 7f5137fff700  1 -- 172.21.15.186:0/2935118583 <== mon.1 v2:172.21.15.186:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x7f512c003770 con 0x7f51401096e0" in cluster log
1 jobs: ['7321536']
suites: ['fixed-2', 'mode/packaged', 'mon_election/classic', 'msgr/async-v2only', 'orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools', 'start', 'tasks/rotate-keys}']

Failure: "/var/log/ceph/cecde0c4-16b2-11ee-9b2e-001a4aab830c/ceph-mon.smithi103.log:2023-06-29T19:31:39.528+0000 7f3ffefa5700 10 mon.smithi103@0(leader).log v203  logging 2023-06-29T19:31:38.576058+0000 mgr.smithi103.ayysig (mgr.14229) 150 : cephadm [WRN] Bind address in nfs.foo.0.0.smithi103.cgqjnw's ganesha conf is defaulting to empty" in cluster log
4 jobs: ['7321504', '7321514', '7321543', '7321541']
suites intersection: []
suites union: ['0-nvme-loop', '1-start', '2-services/nfs-ingress-rgw-bucket', '2-services/nfs-ingress-rgw-user', '2-services/nfs2', '3-final}', 'agent/off', 'mon_election/classic', 'orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/smoke-roleless/{0-distro/ubuntu_20.04', 'orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_3.0', 'task/test_nfs}']

Failure: "/var/log/ceph/d0998354-16b2-11ee-9b2e-001a4aab830c/ceph-mon.c.log:2023-06-29T19:28:50.839+0000 7f75d1345700  7 mon.c@2(synchronizing).log v61 update_from_paxos applying incremental log 60 2023-06-29T19:28:48.856141+0000 mon.a (mon.0) 174 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN)" in cluster log
2 jobs: ['7321507', '7321525']
suites intersection: ['fixed-2', 'mon_election/classic']
suites union: ['0-nvme-loop', 'agent/off', 'fixed-2', 'mode/packaged', 'mon_election/classic', 'msgr/async', 'orch:cephadm/smoke/{0-distro/rhel_8.6_container_tools_rhel8', 'orch:cephadm/with-work/{0-distro/rhel_8.6_container_tools_rhel8', 'start', 'start}', 'tasks/rados_api_tests}']

Failure: "/var/log/ceph/e89b9418-16b4-11ee-9b2e-001a4aab830c/ceph-mon.a.log:2023-06-29T19:47:45.851+0000 7feaf9e3b700  0 log_channel(cluster) log [WRN] : Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY)" in cluster log
1 jobs: ['7321532']
suites: ['2-repo_digest/repo_digest', '3-upgrade/simple', '4-wait', '5-upgrade-ls', 'agent/off', 'mon_election/connectivity}', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04']

Failure: "/var/log/ceph/a2b55be8-16b2-11ee-9b2e-001a4aab830c/ceph-mon.smithi086.log:2023-06-29T19:31:16.938+0000 7fc56cb69700 10 mon.smithi086@0(leader).log v228  logging 2023-06-29T19:31:16.627616+0000 mgr.smithi086.dfxddz (mgr.14221) 171 : cephadm [WRN] Bind address in nfs.foo.0.0.smithi086.ufomeo's ganesha conf is defaulting to empty" in cluster log
1 jobs: ['7321508']
suites: ['0-nvme-loop', '1-start', '2-services/nfs-ingress2', '3-final}', 'orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_rhel8']

Failure: "/var/log/ceph/7724c6ce-16b4-11ee-9b2e-001a4aab830c/ceph-mon.c.log:2023-06-29T19:43:04.202+0000 7fba4f67c700  7 mon.c@2(peon).log v175 update_from_paxos applying incremental log 175 2023-06-29T19:43:03.187599+0000 mon.a (mon.0) 519 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log
1 jobs: ['7321518']
suites: ['fixed-2', 'mode/root', 'mon_election/connectivity', 'msgr/async-v2only', 'orch:cephadm/with-work/{0-distro/rhel_8.6_container_tools_3.0', 'start', 'tasks/rotate-keys}']

Failure: "/var/log/ceph/a7de0f2e-16b3-11ee-9b2e-001a4aab830c/ceph-mon.a.log:2023-06-29T19:37:58.166+0000 7fb4a39b9700  0 log_channel(cluster) log [WRN] : Health check failed: noscrub flag(s) set (OSDMAP_FLAGS)" in cluster log
1 jobs: ['7321509']
suites: ['1-start', '2-thrash', '3-tasks/small-objects', 'fixed-2', 'msgr/async-v2only', 'orch:cephadm/thrash/{0-distro/centos_8.stream_container_tools', 'root}']

Failure: "/var/log/ceph/ee729c50-16b5-11ee-9b2e-001a4aab830c/ceph-mon.smithi007.log:2023-06-29T19:56:12.738+0000 7fc9a5ae3700 10 mon.smithi007@0(leader).log v285  logging 2023-06-29T19:56:12.569889+0000 mgr.smithi026.qnkoum (mgr.24441) 52 : cephadm [WRN] Bind address in nfs.foo.0.1.smithi007.jvudij's ganesha conf is defaulting to empty" in cluster log
1 jobs: ['7321540']
suites: ['1-bootstrap/16.2.5', '1-start', '2-nfs', '3-upgrade-with-workload', '4-final}', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools']