Found 90 jobs Found 5 distinct failure reasons Failure: Command failed on smithi083 with status 1: "sudo cephadm --image quay.ceph.io/ceph-ci/ceph:dc23901172df8c297d89443b9b28803c14d24cd5 -v bootstrap --fsid 1ba55bb2-d009-11ec-8c39-001a4aab830c --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-addrv '[v1:172.21.15.83:6789]' --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring" 50 jobs suites intersection: [] suites union: ['0-nvme-loop', '1-start', '2-ops/repave-all}', '2-ops/rm-zap-add}', '2-ops/rm-zap-flag}', '2-ops/rm-zap-wait}', '2-ops/rmdir-reactivate}', '2-services/basic', '2-services/client-keyring', '2-services/iscsi', '2-services/mirror', '2-services/nfs', '2-services/nfs-ingress', '2-services/nfs-ingress-rgw-bucket', '2-services/nfs-ingress-rgw-user', '2-services/nfs-ingress2', '2-services/nfs2', '2-services/rgw', '2-services/rgw-ingress', '2-thrash', '3-final}', '3-tasks/rados_api_tests', '3-tasks/radosbench', '3-tasks/small-objects', '3-tasks/snaps-few-objects', 'agent/off', 'agent/on', 'base/install', 'cluster/{fixed-3', 'fixed-2', 'mode/packaged', 'mode/root', 'mon_election/classic', 'mon_election/connectivity', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'openstack}', 'orch:cephadm/osds/{0-distro/centos_8.stream_container_tools', 'orch:cephadm/osds/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/osds/{0-distro/rhel_8.5_container_tools_3.0', 'orch:cephadm/osds/{0-distro/rhel_8.5_container_tools_rhel8', 'orch:cephadm/osds/{0-distro/ubuntu_20.04', 'orch:cephadm/rbd_iscsi/{0-single-container-host', 'orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools', 'orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/smoke-roleless/{0-distro/rhel_8.5_container_tools_3.0', 'orch:cephadm/smoke-roleless/{0-distro/rhel_8.5_container_tools_rhel8', 'orch:cephadm/smoke-roleless/{0-distro/ubuntu_20.04', 'orch:cephadm/smoke-singlehost/{0-random-distro$/{rhel_8.5_container_tools_3.0}', 'orch:cephadm/smoke/{0-distro/centos_8.stream_container_tools', 'orch:cephadm/smoke/{0-distro/rhel_8.5_container_tools_rhel8', 'orch:cephadm/smoke/{0-distro/ubuntu_20.04', 'orch:cephadm/thrash/{0-distro/centos_8.stream_container_tools', 'orch:cephadm/thrash/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/thrash/{0-distro/rhel_8.5_container_tools_3.0', 'orch:cephadm/thrash/{0-distro/ubuntu_20.04', 'orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools', 'orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/with-work/{0-distro/rhel_8.5_container_tools_3.0', 'orch:cephadm/with-work/{0-distro/rhel_8.5_container_tools_rhel8', 'orch:cephadm/with-work/{0-distro/ubuntu_20.04', 'orch:cephadm/workunits/{agent/off', 'orch:cephadm/workunits/{agent/on', 'root}', 'start', 'start}', 'task/test_nfs}', 'task/test_orch_cli}', 'tasks/rados_api_tests}', 'tasks/rados_python}', 'workloads/cephadm_iscsi}'] Failure: Command failed on smithi155 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:dc23901172df8c297d89443b9b28803c14d24cd5 -v bootstrap --fsid 2b13c1d2-d00a-11ec-8c39-001a4aab830c --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.155 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring' 16 jobs: ['6829289', '6829282', '6829308', '6829275', '6829332', '6829346', '6829271', '6829297', '6829315', '6829284', '6829321', '6829316', '6829349', '6829339', '6829337', '6829322'] suites intersection: [] suites union: ['0-nvme-loop', '1-start', '2-services/basic', '2-thrash', '3-final}', '3-tasks/radosbench', '3-tasks/small-objects', 'agent/off', 'agent/on', 'fixed-2', 'mode/packaged', 'mode/root', 'mon_election/classic', 'mon_election/connectivity', 'msgr/async', 'msgr/async-v2only', 'orch:cephadm/smoke-singlehost/{0-random-distro$/{centos_8.stream_container_tools}', 'orch:cephadm/smoke/{0-distro/centos_8.stream_container_tools', 'orch:cephadm/smoke/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/smoke/{0-distro/rhel_8.5_container_tools_3.0', 'orch:cephadm/thrash/{0-distro/centos_8.stream_container_tools', 'orch:cephadm/thrash/{0-distro/rhel_8.5_container_tools_rhel8', 'orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools', 'orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools_crun', 'orch:cephadm/with-work/{0-distro/rhel_8.5_container_tools_3.0', 'orch:cephadm/with-work/{0-distro/rhel_8.5_container_tools_rhel8', 'orch:cephadm/with-work/{0-distro/ubuntu_20.04', 'orch:cephadm/workunits/{agent/off', 'orch:cephadm/workunits/{agent/on', 'root}', 'start', 'start}', 'task/test_iscsi_pids_limit/{centos_8.stream_container_tools', 'task/test_orch_cli}', 'tasks/rados_api_tests}', 'tasks/rados_python}', 'test_iscsi_pids_limit}}'] Failure: "2022-05-10T02:22:10.459841+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: nfs, test_orchestrator, mds_autoscaler, cephadm, rook, orchestrator, volumes, dashboard" in cluster log 2 jobs: ['6829273', '6829317'] suites intersection: ['2-node-mgr', 'orchestrator_cli}'] suites union: ['2-node-mgr', 'agent/off', 'agent/on', 'orch:cephadm/orchestrator_cli/{0-random-distro$/{centos_8.stream_container_tools_crun}', 'orch:cephadm/orchestrator_cli/{0-random-distro$/{centos_8.stream_container_tools}', 'orchestrator_cli}'] Failure: Command failed on smithi057 with status 22: "sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5fb194b8-d00b-11ec-8c39-001a4aab830c -e sha1=dc23901172df8c297d89443b9b28803c14d24cd5 -- bash -c 'ceph orch ps'" 14 jobs: ['6829303', '6829327', '6829314', '6829290', '6829354', '6829272', '6829301', '6829312', '6829302', '6829334', '6829286', '6829338', '6829276', '6829353'] suites intersection: [] suites union: ['1-bootstrap/16.2.4', '1-bootstrap/16.2.5', '1-bootstrap/octopus', '1-ranks/1', '1-ranks/2', '1-start', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client', '2-nfs', '2-repo_digest/defaut', '2-repo_digest/repo_digest', '3-inline/no', '3-inline/yes', '3-upgrade-with-workload', '3-upgrade/simple', '3-upgrade/staggered', '4-final}', '4-verify}', '4-verify}}', '4-wait', '5-upgrade-ls', 'agent/off', 'agent/on', 'centos_8.stream_container_tools', 'conf/{client', 'mds', 'mon', 'mon_election/classic}', 'mon_election/connectivity}', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools', 'orch:cephadm/upgrade/{1-start-distro/1-start-centos_8.stream_container-tools', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04-15.2.9', 'osd}', 'overrides/{pg-warn', 'roles', 'syntax', 'tasks/{0-from/pacific', 'tasks/{0-from/v16.2.4', 'whitelist_health', 'whitelist_wrongly_marked_down}'] Failure: Command failed on smithi109 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v15.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5286bc3e-d00e-11ec-8c39-001a4aab830c -e sha1=dc23901172df8c297d89443b9b28803c14d24cd5 -- bash -c \'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)"\'' 3 jobs: ['6829342', '6829328', '6829277'] suites intersection: ['3-upgrade/staggered', '4-wait', '5-upgrade-ls', 'agent/off'] suites union: ['2-repo_digest/defaut', '2-repo_digest/repo_digest', '3-upgrade/staggered', '4-wait', '5-upgrade-ls', 'agent/off', 'mon_election/classic}', 'mon_election/connectivity}', 'orch:cephadm/upgrade/{1-start-distro/1-start-centos_8.stream_container-tools', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04-15.2.9']