archive_path: /home/teuthworker/archive/teuthology-2022-08-14_07:07:02-orch-quincy-distro-default-smithi/6972146 branch: quincy description: orch/cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04-15.2.9 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/classic} email: ceph-qa@ceph.io first_in_suite: false job_id: '6972146' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2022-08-14_07:07:02-orch-quincy-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: quincy ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: true mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 3f62a036422497c2b06717f93312cd5caa471675 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 install: ceph: flavor: default sha1: 3f62a036422497c2b06717f93312cd5caa471675 workunit: branch: quincy sha1: d8f610b9643f0ffafccdb67ab38e9c66321a77ae owner: scheduled_teuthology@teuthology priority: 1000 repo: git://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b - ceph.iscsi.iscsi.a sha1: 3f62a036422497c2b06717f93312cd5caa471675 sleep_before_teardown: 0 suite: orch suite_branch: quincy suite_path: /home/teuthworker/src/git.ceph.com_ceph_d8f610b9643f0ffafccdb67ab38e9c66321a77ae/qa suite_relpath: qa suite_repo: git://git.ceph.com/ceph.git suite_sha1: d8f610b9643f0ffafccdb67ab38e9c66321a77ae targets: smithi131.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDL75j1SRlGHKfIg4Ykg+KnhVs8IWUye6xUGvKeVGQBafTPvTYmYg9K0bJJ11pL9itFChX74qGn4/t4Et+4qLxaqpCQ4vFOeNm3anG+cwlV4Ee5OGtCQV7kifW2540cV+ZwA1c6YdSGI5ssC2D41tBLwXH70xXJHnAPEJhg8dJhej0j0lTsKLEieAgDq0JDlp5Ao13fNisatekGqBOy3Vx1lfCQ8v7nJ32HqG3Jx0935W2LPhJgMF9HB2z4o8GBXsCHNjaHJvzdWoSsIsiuugiWoNrcBoGCYUAJmho7OckXL/PHgcQaL863q8Tm1aNNPTZbaGSRLm9C6x2YIKcS33LjuJKGHTYSN7KueqW2nH05Ln6zLaZaFIllQiSf6pTDbotztr/rNKinGE394W9UU2ZGkX1THFcokzDKtVLz8qB/yezWcQ8UO9YY0QK0F8FlXw+0oQsnJZpuDmkYivrgaWJdugNcA4b7iHjqMqmbF+QHu+AcOPrhv5IzRPQXZRa1BSU= smithi204.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDUPK2Y9IT7noNkoPhI2N5DhrZrG2NGGVOO37ztK+J7LKAVy7hyQJ2G8ggNX7jrkIItBXZ3SRePOXgc5BEbGWSqGKxUw0DIMmZf475ySa//33eM7GrOKxI+dGmUG78D+enu30SOCu1et1genfwQrBVDqIIgE9mUKo7dwGKwvfGCw1/TzU1KzrvTe7i0pv6ehiAqId5V7Q1cqgLxiUVc/Wh0gtMDtLp0UOonr8/UuAFDZENrNaUgIQWzJr3CMd9Hkmsss/h9d/cgAsgywK+j/a0UN2LV6q31a452kCoZNyfHWFUucc9uV2dKiQMi5BaUEcS3KybJlUQvN7juoKPyVxEkdJESfuHrJ4iTE5jqfpQ/DnqQDhQGKpfgne/P/DyvCZLAAa/S/yRRSI6Z1sHO1Gem4pJaTPRg6P18ruWtSBRY9frMZqBankrzPpMBBEiWHA9ZcwJivOdSoXOqcY1SVgmshlKuMZwdpicGXW+AO8AMju1kVTi+RTU7YONpXYjlBbE= tasks: - cephadm: add_mons_via_daemon_add: true allow_ptrace: false avoid_pacific_features: true cephadm_branch: v15.2.9 cephadm_git_url: https://github.com/ceph/ceph image: docker.io/ceph/ceph:v15.2.9 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw r z --placement=2 --port=8000 - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph config set mgr container_image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.r.z - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9e7483cc68a9eb6b54dacbb0bec3bf23a5d32425 timestamp: 2022-08-14_07:07:02 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.1713