2023-08-15T21:41:25.463 INFO:root:teuthology version: 0.0.1.dev185+g449a1bc 2023-08-15T21:41:25.464 INFO:teuthology.results:subset: '111/120000' 2023-08-15T21:41:25.464 INFO:teuthology.results:seed: '17' 2023-08-15T23:11:36.647 INFO:root:teuthology version: 0.0.1.dev185+g449a1bc 2023-08-15T23:11:36.669 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2023-08-16T04:14:32.673 INFO:teuthology.results:Tests finished! gathering results... 2023-08-16T04:14:42.763 INFO:scrape:Found 171 jobs 2023-08-16T04:14:42.764 WARNING:scrape:Missing teuthology log /home/teuthworker/archive/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368833/teuthology.log 2023-08-16T04:15:02.722 INFO:scrape:Found 12 distinct failure reasons 2023-08-16T04:15:02.723 INFO:scrape:Dead: None 2023-08-16T04:15:02.724 INFO:scrape:75 jobs 2023-08-16T04:15:02.724 INFO:scrape: 2023-08-16T04:15:02.724 INFO:scrape:Failure: "2023-08-15T23:22:53.374583+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log 2023-08-16T04:15:02.725 INFO:scrape:69 jobs 2023-08-16T04:15:02.726 INFO:scrape:suites intersection: ['conf/{client', 'mds', 'mon', 'osd}'] 2023-08-16T04:15:02.726 INFO:scrape:suites union: ['2-workunit/cfuse_workunit_snaptests}}', '2-workunit/cfuse_workunit_suites_fsstress}}', '2-workunit/cfuse_workunit_suites_pjd}}', '2-workunit/fs/snaps}}', '2-workunit/fs/trivial_sync}}', '2-workunit/iozone}}', '2-workunit/suites/ffsb}}', '2-workunit/suites/fsstress}}', '2-workunit/suites/pjd}}', 'clusters/1-mds-1-client-coloc', 'clusters/1-mds-2-client', 'clusters/1-mds-3-client', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'clusters/fixed-2-ucephfs', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'distros/ubuntu_latest', 'fs/32bits/{begin', 'fs/functional/{begin', 'fs/libcephfs/{begin', 'fs/mixed-clients/{begin', 'fs/multiclient/{begin', 'fs/multifs/{begin', 'fs/permission/{begin', 'fs/shell/{begin', 'fs/snaps/{begin', 'fs/thrash/multifs/{begin', 'fs/thrash/workloads/{begin', 'fs/traceless/{begin', 'fs/verify/{begin', 'fs/volumes/{begin', 'fs/workload/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'k-testing}', 'kclient-overrides/{distro/testing/{flavor/ubuntu_latest', 'mds', 'mon', 'mon-debug', 'mon-debug}', 'mount/fuse', 'mount/kclient/{mount', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'ms_mode/{crc}', 'ms_mode/{legacy}', 'ms_mode/{secure}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd-asserts}', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{faked-ino', 'overrides/{frag', 'overrides/{ignorelist_health', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'races', 'ranks/1', 'ranks/3', 'ranks/5', 'rhel_8}', 'scrub/no', 'scrub/yes', 'session_timeout', 'session_timeout}', 'standby-replay', 'tasks/auto-repair}', 'tasks/cephfs-shell}', 'tasks/cfuse_workunit_misc}', 'tasks/cfuse_workunit_suites_blogbench', 'tasks/cfuse_workunit_suites_dbench', 'tasks/cfuse_workunit_suites_ffsb', 'tasks/cfuse_workunit_suites_fsstress}', 'tasks/cfuse_workunit_suites_pjd}', 'tasks/client-readahead}', 'tasks/client-recovery}', 'tasks/client}', 'tasks/damage}', 'tasks/data-scan}', 'tasks/dbench', 'tasks/exports}', 'tasks/failover}', 'tasks/fragment}', 'tasks/journal-repair}', 'tasks/kernel_cfuse_workunits_untarbuild_blogbench}', 'tasks/libcephfs_python}', 'tasks/mdtest}', 'tasks/metrics}', 'tasks/multifs-auth}', 'tasks/multimds_misc}', 'tasks/openfiletable}', 'tasks/recovery-fs}', 'tasks/scrub}', 'tasks/sessionmap}', 'tasks/snapshots}', 'tasks/truncate_delay}', 'tasks/volume-client}', 'tasks/volumes/{overrides', 'tasks/workunit/quota}', 'tasks/workunit/snaps}', 'tasks/{0-check-counter', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'test/clone}}', 'thrashosds-health}', 'traceless/50pc}', 'validater/valgrind}', 'workunit/direct_io}', 'workunit/kernel_untar_build}', 'workunit/suites/blogbench}', 'workunit/suites/fsstress}', 'workunit/suites/fsx}', 'workunit/suites/iozone}', 'workunit/suites/pjd}', 'wsync/{no}}', 'wsync/{yes}}'] 2023-08-16T04:15:02.727 INFO:scrape: 2023-08-16T04:15:02.727 INFO:scrape:Failure: "2023-08-15T23:11:29.954503+0000 mon.a (mon.0) 64 : cluster [WRN] Health check failed: Module 'devicehealth' has failed dependency: cannot import name 'MgrDBNotReady' (MGR_MODULE_DEPENDENCY)" in cluster log 2023-08-16T04:15:02.727 INFO:scrape:6 jobs: ['7368920', '7368910', '7368905', '7368917', '7368909', '7368907'] 2023-08-16T04:15:02.728 INFO:scrape:suites intersection: [] 2023-08-16T04:15:02.728 INFO:scrape:suites union: ['2-three-per-cluster}', 'centos_latest', 'cephfs-mirror/one-per-cluster', 'cephfs-mirror/{1-volume-create-rm', 'clients/mirror', 'clients/{mirror}', 'cluster/1-node', 'cluster/{1-node}', 'clusters/1-node-1-mds-1-osd', 'clusters/small-cluster', 'conf/{client', 'debug', 'distro/{rhel_8}', 'fs/bugs/client_trim_caps/{begin', 'fs/full/{begin', 'fs/mirror-ha/{begin', 'fs/mirror/{begin', 'fs/top/{begin', 'fs/valgrind/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mirror/{cephfs-mirror/one-per-cluster', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides', 'overrides/ignorelist_health', 'overrides/whitelist_health', 'overrides/{ignorelist_health', 'overrides/{whitelist_health}', 'supported-random-distro$/{rhel_8}', 'supported-random-distros$/{centos_8}', 'supported-random-distros$/{rhel_8}', 'tasks/fstop}', 'tasks/mgr-osd-full}', 'tasks/mirror}', 'tasks/mirror}}', 'tasks/trim-i24137}', 'workloads/cephfs-mirror-ha-workunit}'] 2023-08-16T04:15:02.728 INFO:scrape: 2023-08-16T04:15:02.729 INFO:scrape:Failure: timeout expired in wait_until_healthy 2023-08-16T04:15:02.729 INFO:scrape:4 jobs: ['7368891', '7368919', '7368973', '7368906'] 2023-08-16T04:15:02.729 INFO:scrape:suites intersection: [] 2023-08-16T04:15:02.729 INFO:scrape:suites union: ['1-client', '1-mount', '2-upgrade', '2-workload/dbench', '3-client-upgrade', '3-compat_client/no}}', '3-compat_client/pacific}}', '4-compat_client', '5-client-sanity}}', 'centos_latest', 'clusters/1-mds-2-client-micro', 'conf/{client', 'distro/centos_latest}', 'fs/cephadm/multivolume/{0-start', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'multimds/no', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/{0-nautilus'] 2023-08-16T04:15:02.730 INFO:scrape: 2023-08-16T04:15:02.730 INFO:scrape:Failure: ['/dev/vg_nvme: already exists in filesystem', "Creating volume group 'vg_nvme' failed", "Run `vgcreate --help' for more information."] 2023-08-16T04:15:02.730 INFO:scrape:3 jobs: ['7368950', '7368820', '7368964'] 2023-08-16T04:15:02.731 INFO:scrape:suites intersection: ['conf/{client', 'distro/{centos_8}', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'osd}'] 2023-08-16T04:15:02.731 INFO:scrape:suites union: ['2-workunit/cfuse_workunit_trivial_sync}}', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin', 'fs/thrash/multifs/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'msgr-failures/osd-mds-delay', 'multifs', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{frag', 'overrides/{ignorelist_health', 'session_timeout', 'tasks/asok_dump_tree}', 'tasks/client-limits}', 'tasks/{1-thrash/mon', 'thrashosds-health}'] 2023-08-16T04:15:02.731 INFO:scrape: 2023-08-16T04:15:02.732 INFO:scrape:Failure: Failed to reconnect to smithi039 2023-08-16T04:15:02.732 INFO:scrape:2 jobs: ['7368869', '7368898'] 2023-08-16T04:15:02.732 INFO:scrape:suites intersection: ['conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health'] 2023-08-16T04:15:02.732 INFO:scrape:suites union: ['1-client', '2-upgrade', '3-client-upgrade', '4-compat_client', '5-client-sanity}}', 'centos_latest', 'clusters/1-mds-2-client-micro', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin', 'fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'multimds/yes', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/snap_schedule_snapdir}', 'tasks/{0-nautilus'] 2023-08-16T04:15:02.733 INFO:scrape: 2023-08-16T04:15:02.733 INFO:scrape:Failure: ['See stdout/stderr for the exact error', 'The module failed to execute correctly, you probably need to set the interpreter.'] 2023-08-16T04:15:02.733 INFO:scrape:1 jobs: ['7368892'] 2023-08-16T04:15:02.734 INFO:scrape:suites: ['2-workunit/ffsb}}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{centos_8}', 'fs/thrash/multifs/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'msgr-failures/none', 'multifs', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{frag', 'session_timeout', 'tasks/{1-thrash/mon', 'thrashosds-health}'] 2023-08-16T04:15:02.734 INFO:scrape: 2023-08-16T04:15:02.734 INFO:scrape:Dead: b"TypeError: argument of type 'NoneType' is not iterable" 2023-08-16T04:15:02.735 INFO:scrape:2 jobs: ['7368927', '7368896'] 2023-08-16T04:15:02.735 INFO:scrape:suites intersection: ['clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/thrash/workloads/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'msgr-failures/osd-mds-delay', 'osd}', 'overrides/{frag', 'prefetch_dirfrags/no', 'races', 'session_timeout', 'thrashosds-health}'] 2023-08-16T04:15:02.735 INFO:scrape:suites union: ['2-workunit/fs/snaps}}', '2-workunit/suites/fsstress}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/thrash/workloads/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/fuse', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{frag', 'prefetch_dirfrags/no', 'races', 'ranks/1', 'ranks/5', 'session_timeout', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'thrashosds-health}'] 2023-08-16T04:15:02.735 INFO:scrape: 2023-08-16T04:15:02.736 INFO:scrape:Failure: [Errno None] Unable to connect to port 22 on 172.21.15.131 2023-08-16T04:15:02.736 INFO:scrape:1 jobs: ['7368875'] 2023-08-16T04:15:02.736 INFO:scrape:suites: ['clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/multifs/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'mount/fuse', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/multifs-auth}'] 2023-08-16T04:15:02.737 INFO:scrape: 2023-08-16T04:15:02.737 INFO:scrape:Failure: timed out 2023-08-16T04:15:02.737 INFO:scrape:1 jobs: ['7368912'] 2023-08-16T04:15:02.738 INFO:scrape:suites: ['clusters/1-mds-3-client', 'conf/{client', 'distros/ubuntu_latest', 'fs/multiclient/{begin', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs_misc_tests}'] 2023-08-16T04:15:02.738 INFO:scrape: 2023-08-16T04:15:02.738 INFO:scrape:Failure: Command failed on smithi039 with status 1: 'sudo yum install -y kernel' 2023-08-16T04:15:02.738 INFO:scrape:1 jobs: ['7368873'] 2023-08-16T04:15:02.739 INFO:scrape:suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/pool-perm}'] 2023-08-16T04:15:02.739 INFO:scrape: 2023-08-16T04:15:02.739 INFO:scrape:Failure: failed to install new kernel version within timeout 2023-08-16T04:15:02.740 INFO:scrape:1 jobs: ['7368903'] 2023-08-16T04:15:02.740 INFO:scrape:suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/strays}'] 2023-08-16T04:15:02.740 INFO:scrape: 2023-08-16T04:15:02.746 INFO:teuthology.results:Sending results to yweinste@redhat.com: Test Run: yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi ================================================================= info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/ logs: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/ failed: 86 dead: 80 running: 0 waiting: 0 queued: 0 passed: 5 Fail ================================================================= [7368817] fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_dbench traceless/50pc} ----------------------------------------------------------------- time: 00:17:34 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368817/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368817/ sentry: https://sentry.ceph.com/organizations/ceph/?query=9d06d6f514d74db6a64a707220db882c "2023-08-15T22:04:09.069599+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368824] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/damage} ----------------------------------------------------------------- time: 00:17:16 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368824/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368824/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ec27c77531744130b62cc13da1a902b5 "2023-08-15T23:09:46.309989+0000 mgr.x (mgr.4123) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368829] fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} ----------------------------------------------------------------- time: 00:19:10 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368829/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368829/ sentry: https://sentry.ceph.com/organizations/ceph/?query=1ecccaf688134bf6a2c0510aac1ba9cc "2023-08-15T22:12:50.440455+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368841] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/fragment} ----------------------------------------------------------------- time: 00:17:17 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368841/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368841/ sentry: https://sentry.ceph.com/organizations/ceph/?query=6c3b60a28b2f40e0adc470bc4443b1f7 "2023-08-15T23:14:25.649383+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368842] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} ----------------------------------------------------------------- time: 00:19:28 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368842/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368842/ sentry: https://sentry.ceph.com/organizations/ceph/?query=663fa3a3e25b455397bbeea4e088b8c2 "2023-08-15T23:18:27.062233+0000 mgr.y (mgr.4110) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368843] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/journal-repair} ----------------------------------------------------------------- time: 00:19:17 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368843/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368843/ sentry: https://sentry.ceph.com/organizations/ceph/?query=49bc5c6504b441d191650c00cbdd38b7 "2023-08-15T22:16:29.767896+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368844] fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} ----------------------------------------------------------------- time: 00:18:13 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368844/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368844/ sentry: https://sentry.ceph.com/organizations/ceph/?query=80d7a158801b414ea31037a90e04c370 "2023-08-15T22:16:51.067810+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368845] fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/failover} ----------------------------------------------------------------- time: 00:20:15 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368845/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368845/ sentry: https://sentry.ceph.com/organizations/ceph/?query=61a9a2dadaa443b2ab1738e8951de02a "2023-08-15T22:22:57.065660+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368846] fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} ----------------------------------------------------------------- time: 00:17:31 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368846/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368846/ sentry: https://sentry.ceph.com/organizations/ceph/?query=535c2a8b53554cffa8724002ab3706bd "2023-08-15T22:17:36.293743+0000 mgr.y (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368862] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/iozone} wsync/{no}} ----------------------------------------------------------------- time: 00:20:22 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368862/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368862/ sentry: https://sentry.ceph.com/organizations/ceph/?query=4393f36c16344c608137d30b4d7918fb "2023-08-15T22:24:45.830409+0000 mgr.z (mgr.4111) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368863] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/metrics} ----------------------------------------------------------------- time: 00:21:13 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368863/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368863/ sentry: https://sentry.ceph.com/organizations/ceph/?query=93c8c3faf86b477ea1b068ef7abce88d "2023-08-15T22:23:28.339451+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368864] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/trivial_sync}} ----------------------------------------------------------------- time: 00:22:26 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368864/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368864/ sentry: https://sentry.ceph.com/organizations/ceph/?query=35cfabcb3c744ed0acf5d9103a93a728 "2023-08-15T22:25:58.054125+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368865] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/multimds_misc} ----------------------------------------------------------------- time: 00:22:26 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368865/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368865/ sentry: https://sentry.ceph.com/organizations/ceph/?query=2e4894cdd8b24d07a8dcad1a846be656 "2023-08-15T22:30:40.104667+0000 mgr.x (mgr.4114) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368866] fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} ----------------------------------------------------------------- time: 00:18:59 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368866/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368866/ sentry: https://sentry.ceph.com/organizations/ceph/?query=161777f92aad48958d45ec4361d3de25 "2023-08-15T22:25:38.652808+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368867] fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{centos_8} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/libcephfs_python} ----------------------------------------------------------------- time: 00:19:03 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368867/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368867/ sentry: https://sentry.ceph.com/organizations/ceph/?query=311208ffe5db4c399fe08752047da7a3 "2023-08-15T22:27:36.896498+0000 mgr.x (mgr.4112) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368868] fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} ----------------------------------------------------------------- time: 00:18:40 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368868/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368868/ sentry: https://sentry.ceph.com/organizations/ceph/?query=6f44f3fa80614d9bb47ae0a347d0ad85 "2023-08-15T22:25:29.561470+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368869] fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-nautilus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368869/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368869/ sentry: https://sentry.ceph.com/organizations/ceph/?query=639422520c704c299f1ba88846824a8c Failed to reconnect to smithi039 [7368870] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_pjd}} ----------------------------------------------------------------- time: 00:23:59 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368870/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368870/ sentry: https://sentry.ceph.com/organizations/ceph/?query=3dbc2ad318a548f981e458a6422eb20e "2023-08-15T22:32:13.498063+0000 mgr.y (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368871] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/openfiletable} ----------------------------------------------------------------- time: 00:20:04 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368871/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368871/ sentry: https://sentry.ceph.com/organizations/ceph/?query=d868d05c040f48d38a74c4a60ebe5fb7 "2023-08-15T23:28:43.745271+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368872] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse ms_mode/{secure} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/pjd} wsync/{yes}} ----------------------------------------------------------------- time: 00:18:13 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368872/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368872/ sentry: https://sentry.ceph.com/organizations/ceph/?query=2c3ae22804f54ccfa35506c531652fc5 "2023-08-15T22:28:53.214052+0000 mgr.z (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368873] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/pool-perm} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368873/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368873/ sentry: https://sentry.ceph.com/organizations/ceph/?query=5be3c40a36e547dfb1cae5f76bb42622 Command failed on smithi039 with status 1: 'sudo yum install -y kernel' [7368875] fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/multifs-auth} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368875/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368875/ sentry: https://sentry.ceph.com/organizations/ceph/?query=5de4bfcbb4614bf79d568537ca79f109 [Errno None] Unable to connect to port 22 on 172.21.15.131 [7368876] fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} ----------------------------------------------------------------- time: 00:18:35 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368876/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368876/ sentry: https://sentry.ceph.com/organizations/ceph/?query=2b2b8322a68f4a1fb3ce9d19d8ca2be0 "2023-08-15T23:26:43.574780+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368883] fs/multiclient/{begin clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/mdtest} ----------------------------------------------------------------- time: 00:17:57 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368883/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368883/ sentry: https://sentry.ceph.com/organizations/ceph/?query=8a527da79a3246a9b20ac1ccdf629ccc "2023-08-15T22:32:29.665338+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368884] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/{crc} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/direct_io} wsync/{no}} ----------------------------------------------------------------- time: 00:23:59 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368884/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368884/ sentry: https://sentry.ceph.com/organizations/ceph/?query=42ae1c8c898b4604a5191e6d5d1ab6c6 "2023-08-15T22:38:45.149054+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368885] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/recovery-fs} ----------------------------------------------------------------- time: 00:23:07 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368885/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368885/ sentry: https://sentry.ceph.com/organizations/ceph/?query=66094fcee5b042269f2d6e4c098b1c6a "2023-08-15T22:37:17.729737+0000 mgr.y (mgr.4110) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368887] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:19:54 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368887/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368887/ sentry: https://sentry.ceph.com/organizations/ceph/?query=4c58c47f70ce4863b4b362ef28321d9f "2023-08-15T22:37:07.461001+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368888] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/scrub} ----------------------------------------------------------------- time: 00:20:34 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368888/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368888/ sentry: https://sentry.ceph.com/organizations/ceph/?query=eed02cf3530a41c5bb7958636937b383 "2023-08-15T22:39:14.314150+0000 mgr.x (mgr.4101) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368889] fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} ----------------------------------------------------------------- time: 00:19:24 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368889/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368889/ sentry: https://sentry.ceph.com/organizations/ceph/?query=768cbc3ef4b84000aab04bf4a402073e "2023-08-15T22:34:46.102860+0000 mgr.x (mgr.4115) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368890] fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} ----------------------------------------------------------------- time: 00:19:45 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368890/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368890/ sentry: https://sentry.ceph.com/organizations/ceph/?query=4af3a45b14a544f1aba2f7ed14983067 "2023-08-15T22:36:12.711471+0000 mgr.x (mgr.4114) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368891] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-nautilus 1-client 2-upgrade 3-compat_client/pacific}} ----------------------------------------------------------------- time: 00:28:01 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368891/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368891/ sentry: https://sentry.ceph.com/organizations/ceph/?query=7cd347b62fbb4f4cb37d53c8c5275eea timeout expired in wait_until_healthy [7368892] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/ffsb}} ----------------------------------------------------------------- time: 00:13:20 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368892/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368892/ sentry: https://sentry.ceph.com/organizations/ceph/?query=1c062ff05ff043cfb43550df05e66b89 ['See stdout/stderr for the exact error', 'The module failed to execute correctly, you probably need to set the interpreter.'] [7368893] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/sessionmap} ----------------------------------------------------------------- time: 00:18:02 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368893/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368893/ sentry: https://sentry.ceph.com/organizations/ceph/?query=23f7799dc6114c85aa172151a8f11c91 "2023-08-15T22:36:56.626554+0000 mgr.x (mgr.4124) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368898] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/snap_schedule_snapdir} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368898/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368898/ sentry: https://sentry.ceph.com/organizations/ceph/?query=76e399a04842435b92707b4e86b5bcf5 Failed to reconnect to smithi132 [7368899] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/iozone}} ----------------------------------------------------------------- time: 00:20:32 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368899/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368899/ sentry: https://sentry.ceph.com/organizations/ceph/?query=d53d84bc58234ed8b50c90292ed86548 "2023-08-15T22:42:02.591482+0000 mgr.y (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368900] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/snapshots} ----------------------------------------------------------------- time: 00:19:35 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368900/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368900/ sentry: https://sentry.ceph.com/organizations/ceph/?query=c2f5a6109a5745b98e56134b3d7b0873 "2023-08-15T22:45:07.139846+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368901] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/kernel_untar_build} wsync/{no}} ----------------------------------------------------------------- time: 00:20:43 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368901/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368901/ sentry: https://sentry.ceph.com/organizations/ceph/?query=4a8b30e26b704ebebd4a30f4ee7b9ccf "2023-08-15T22:48:25.453427+0000 mgr.z (mgr.4121) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368902] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/ffsb}} ----------------------------------------------------------------- time: 00:20:24 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368902/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368902/ sentry: https://sentry.ceph.com/organizations/ceph/?query=434ad23a3996441fb5333e589f73b877 "2023-08-15T22:46:12.208674+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368903] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/strays} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368903/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368903/ sentry: https://sentry.ceph.com/organizations/ceph/?query=1c003896f3d7429da3d7287c58e335db failed to install new kernel version within timeout [7368905] fs/bugs/client_trim_caps/{begin centos_latest clusters/small-cluster conf/{client mds mon osd} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/trim-i24137} ----------------------------------------------------------------- time: 00:19:24 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368905/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368905/ sentry: https://sentry.ceph.com/organizations/ceph/?query=33d4309430224c299bccc03c95737b03 "2023-08-15T22:46:11.540637+0000 mon.a (mon.0) 65 : cluster [WRN] Health check failed: Module 'devicehealth' has failed dependency: cannot import name 'MgrDBNotReady' (MGR_MODULE_DEPENDENCY)" in cluster log [7368906] fs/cephadm/multivolume/{0-start 1-mount 2-workload/dbench distro/centos_latest} ----------------------------------------------------------------- time: 00:23:49 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368906/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368906/ sentry: https://sentry.ceph.com/organizations/ceph/?query=9f2741656d3f4729880cadfb5db280ba timeout expired in wait_until_healthy [7368907] fs/full/{begin clusters/1-node-1-mds-1-osd conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-bitmap overrides overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mgr-osd-full} ----------------------------------------------------------------- time: 00:24:17 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368907/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368907/ sentry: https://sentry.ceph.com/organizations/ceph/?query=300f6ae5f054475a8878032ecdb25fc2 "2023-08-15T22:46:23.103165+0000 mon.a (mon.0) 40 : cluster [WRN] Health check failed: Module 'devicehealth' has failed dependency: cannot import name 'MgrDBNotReady' (MGR_MODULE_DEPENDENCY)" in cluster log [7368908] fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{centos_8} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client} ----------------------------------------------------------------- time: 00:23:29 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368908/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368908/ sentry: https://sentry.ceph.com/organizations/ceph/?query=c18c8a826b7e403c9b709c330fb0e0cf "2023-08-15T22:55:07.203564+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368909] fs/mirror/{begin cephfs-mirror/one-per-cluster clients/{mirror} cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/{whitelist_health} supported-random-distros$/{centos_8} tasks/mirror} ----------------------------------------------------------------- time: 00:18:39 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368909/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368909/ sentry: https://sentry.ceph.com/organizations/ceph/?query=bbf518eb9f264b02956f55128df27f6c "2023-08-15T22:48:07.606425+0000 mon.a (mon.0) 68 : cluster [WRN] Health check failed: Module 'devicehealth' has failed dependency: cannot import name 'MgrDBNotReady' (MGR_MODULE_DEPENDENCY)" in cluster log [7368910] fs/mirror-ha/{begin cephfs-mirror/{1-volume-create-rm 2-three-per-cluster} clients/{mirror} cluster/{1-node} objectstore/bluestore-bitmap overrides/{whitelist_health} supported-random-distro$/{rhel_8} workloads/cephfs-mirror-ha-workunit} ----------------------------------------------------------------- time: 00:20:31 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368910/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368910/ sentry: https://sentry.ceph.com/organizations/ceph/?query=4be111f7d17147eab13400469fa0a041 "2023-08-15T22:48:22.528156+0000 mon.a (mon.0) 68 : cluster [WRN] Health check failed: Module 'devicehealth' has failed dependency: cannot import name 'MgrDBNotReady' (MGR_MODULE_DEPENDENCY)" in cluster log [7368912] fs/multiclient/{begin clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cephfs_misc_tests} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368912/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368912/ sentry: https://sentry.ceph.com/organizations/ceph/?query=3e5385ea217845b0a2caed906d389260 timed out [7368913] fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/multifs-auth} ----------------------------------------------------------------- time: 00:18:17 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368913/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368913/ sentry: https://sentry.ceph.com/organizations/ceph/?query=5bf179a422bd476b84f65f2b09f4b7f8 "2023-08-15T22:51:30.648213+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368914] fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} ----------------------------------------------------------------- time: 00:19:31 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368914/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368914/ sentry: https://sentry.ceph.com/organizations/ceph/?query=64be67bd20b64785ae04978dc863214f "2023-08-15T22:54:06.070552+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368915] fs/shell/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/cephfs-shell} ----------------------------------------------------------------- time: 00:17:10 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368915/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368915/ sentry: https://sentry.ceph.com/organizations/ceph/?query=49c7f9fbfd0d450f89b9612c4fb3eafb "2023-08-15T22:54:26.920558+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368916] fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} ----------------------------------------------------------------- time: 00:19:27 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368916/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368916/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ed3f346ce2ab47a0b60c9701fb33bf3c "2023-08-15T22:56:56.179698+0000 mgr.y (mgr.4110) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368917] fs/top/{begin cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/ignorelist_health supported-random-distros$/{rhel_8} tasks/fstop} ----------------------------------------------------------------- time: 00:20:07 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368917/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368917/ sentry: https://sentry.ceph.com/organizations/ceph/?query=005309c0ef384d87944efdfae43fb628 "2023-08-15T22:55:53.223592+0000 mon.a (mon.0) 64 : cluster [WRN] Health check failed: Module 'devicehealth' has failed dependency: cannot import name 'MgrDBNotReady' (MGR_MODULE_DEPENDENCY)" in cluster log [7368918] fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_blogbench traceless/50pc} ----------------------------------------------------------------- time: 00:17:57 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368918/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368918/ sentry: https://sentry.ceph.com/organizations/ceph/?query=4d2aff4b733244b088f3fcb1d962f117 "2023-08-15T22:57:19.784247+0000 mgr.y (mgr.4120) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368919] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-nautilus 1-client 2-upgrade 3-compat_client/no}} ----------------------------------------------------------------- time: 00:35:51 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368919/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368919/ sentry: https://sentry.ceph.com/organizations/ceph/?query=44ee7460b7c64c1eb390fbc76a17b117 timeout expired in wait_until_healthy [7368920] fs/valgrind/{begin centos_latest debug mirror/{cephfs-mirror/one-per-cluster clients/mirror cluster/1-node mount/fuse overrides/whitelist_health tasks/mirror}} ----------------------------------------------------------------- time: 00:32:38 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368920/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368920/ sentry: https://sentry.ceph.com/organizations/ceph/?query=cb61a7fa130e4835ac550902531daad2 "2023-08-15T23:11:29.954503+0000 mon.a (mon.0) 64 : cluster [WRN] Health check failed: Module 'devicehealth' has failed dependency: cannot import name 'MgrDBNotReady' (MGR_MODULE_DEPENDENCY)" in cluster log [7368921] fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volume-client} ----------------------------------------------------------------- time: 00:22:50 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368921/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368921/ sentry: https://sentry.ceph.com/organizations/ceph/?query=df94cd0af33f457ca31a56f78f388688 "2023-08-15T23:03:46.093738+0000 mgr.x (mgr.4114) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368922] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_snaptests}} ----------------------------------------------------------------- time: 00:18:15 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368922/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368922/ sentry: https://sentry.ceph.com/organizations/ceph/?query=44a92a74b9404e38b2c156366babaa2c "2023-08-15T22:58:54.329745+0000 mgr.y (mgr.4113) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368925] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/fuse ms_mode/{crc} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/blogbench} wsync/{yes}} ----------------------------------------------------------------- time: 00:20:13 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368925/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368925/ sentry: https://sentry.ceph.com/organizations/ceph/?query=1a002a33ccd44ddd88be98a7ce87390f "2023-08-15T23:01:12.486137+0000 mgr.x (mgr.4110) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368926] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/truncate_delay} ----------------------------------------------------------------- time: 00:22:49 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368926/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368926/ sentry: https://sentry.ceph.com/organizations/ceph/?query=77e202086ab74c0eb0834ca8a22f2381 "2023-08-15T23:06:38.805711+0000 mgr.x (mgr.4112) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368930] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_fsstress}} ----------------------------------------------------------------- time: 00:19:56 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368930/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368930/ sentry: https://sentry.ceph.com/organizations/ceph/?query=a8bf1560a93846548b3f51c6f5800f76 "2023-08-15T23:02:00.875362+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368931] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/workunit/quota} ----------------------------------------------------------------- time: 00:17:23 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368931/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368931/ sentry: https://sentry.ceph.com/organizations/ceph/?query=34b39bb2ad15447cb7609a9af6ee60c4 "2023-08-16T00:02:18.408888+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368936] fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} ----------------------------------------------------------------- time: 00:21:38 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368936/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368936/ sentry: https://sentry.ceph.com/organizations/ceph/?query=078a8b1248b04eb285f93a56a90abdab "2023-08-15T23:13:46.906344+0000 mgr.y (mgr.4101) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368937] fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} ----------------------------------------------------------------- time: 00:19:08 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368937/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368937/ sentry: https://sentry.ceph.com/organizations/ceph/?query=c1e774fc86c849778afbdc97a52e1222 "2023-08-15T23:11:21.995012+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368947] fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/failover} ----------------------------------------------------------------- time: 00:19:17 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368947/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368947/ sentry: https://sentry.ceph.com/organizations/ceph/?query=0008117018894e979bb055ad85df84af "2023-08-15T23:12:14.054189+0000 mgr.y (mgr.4111) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368952] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/auto-repair} ----------------------------------------------------------------- time: 00:21:22 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368952/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368952/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ef8c803f0512459eb4ffde81a0cec946 "2023-08-15T23:16:35.649327+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368965] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 00:23:30 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368965/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368965/ sentry: https://sentry.ceph.com/organizations/ceph/?query=187a44bcfa354221a49b405653d222a8 "2023-08-15T23:24:04.445499+0000 mgr.x (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368966] fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/clone}} ----------------------------------------------------------------- time: 00:22:26 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368966/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368966/ sentry: https://sentry.ceph.com/organizations/ceph/?query=a594366f98b741c58abd7d138d675e4d "2023-08-15T23:23:04.493982+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368967] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/fuse ms_mode/{secure} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsstress} wsync/{no}} ----------------------------------------------------------------- time: 00:24:31 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368967/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368967/ sentry: https://sentry.ceph.com/organizations/ceph/?query=6fccbcaf57bf4be28e9170025c16b1d4 "2023-08-15T23:29:29.661138+0000 mgr.z (mgr.4106) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368968] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client-readahead} ----------------------------------------------------------------- time: 00:19:11 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368968/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368968/ sentry: https://sentry.ceph.com/organizations/ceph/?query=592acd403acd44cfbdd8665a9da7b38e "2023-08-15T23:22:53.374583+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368969] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/iozone}} ----------------------------------------------------------------- time: 00:21:04 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368969/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368969/ sentry: https://sentry.ceph.com/organizations/ceph/?query=7b0ec8a9905c41cc85e81604d0d092e8 "2023-08-15T23:23:13.569398+0000 mgr.x (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368970] fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_untarbuild_blogbench} ----------------------------------------------------------------- time: 00:20:35 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368970/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368970/ sentry: https://sentry.ceph.com/organizations/ceph/?query=1ac9bbee6d8b4f23b42668f42c302390 "2023-08-15T23:24:58.795762+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368971] fs/multiclient/{begin clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/mdtest} ----------------------------------------------------------------- time: 00:19:30 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368971/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368971/ sentry: https://sentry.ceph.com/organizations/ceph/?query=8e7c0f515bbd418f97284259d4318629 "2023-08-15T23:26:48.633546+0000 mgr.y (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368972] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client-recovery} ----------------------------------------------------------------- time: 00:22:41 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368972/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368972/ sentry: https://sentry.ceph.com/organizations/ceph/?query=03a84e1066374aa281276c112129b522 "2023-08-15T23:29:37.788873+0000 mgr.x (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368973] fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-nautilus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} ----------------------------------------------------------------- time: 00:25:45 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368973/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368973/ sentry: https://sentry.ceph.com/organizations/ceph/?query=14482a045661420181c63a45cbd65892 timeout expired in wait_until_healthy [7368974] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/ffsb}} ----------------------------------------------------------------- time: 00:20:58 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368974/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368974/ sentry: https://sentry.ceph.com/organizations/ceph/?query=5ce75f4ba7d34680bace2c1454092b3b "2023-08-15T23:27:09.477099+0000 mgr.y (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368975] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/damage} ----------------------------------------------------------------- time: 00:20:56 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368975/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368975/ sentry: https://sentry.ceph.com/organizations/ceph/?query=a6c30aa46e0e45a7b9b9f9754ab10a40 "2023-08-15T23:27:31.162187+0000 mgr.y (mgr.4114) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368976] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{no}} ----------------------------------------------------------------- time: 00:20:44 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368976/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368976/ sentry: https://sentry.ceph.com/organizations/ceph/?query=de3966cf178b4f9ebe4a21499ec90995 "2023-08-15T23:26:36.781784+0000 mgr.z (mgr.4106) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368977] fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} ----------------------------------------------------------------- time: 00:18:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368977/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368977/ sentry: https://sentry.ceph.com/organizations/ceph/?query=92108c88862c4e3494126e18fb5625d2 "2023-08-15T23:23:08.244257+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368978] fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/multifs-auth} ----------------------------------------------------------------- time: 00:22:42 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368978/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368978/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ca75acd3b25a4454b31ed6a003b92bfa "2023-08-15T23:29:53.627196+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368979] fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} ----------------------------------------------------------------- time: 00:17:58 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368979/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368979/ sentry: https://sentry.ceph.com/organizations/ceph/?query=5aa6dae85bfc4840acec2dafeca3f6ad "2023-08-15T23:26:43.986942+0000 mgr.y (mgr.4114) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368980] fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} ----------------------------------------------------------------- time: 00:23:28 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368980/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368980/ sentry: https://sentry.ceph.com/organizations/ceph/?query=b78738e68bb445909ee240b379f73e97 "2023-08-15T23:32:45.494503+0000 mgr.y (mgr.4101) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368981] fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_ffsb traceless/50pc} ----------------------------------------------------------------- time: 00:18:52 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368981/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368981/ sentry: https://sentry.ceph.com/organizations/ceph/?query=3c8d72ece595470a82dfee99f63f5c93 "2023-08-15T23:25:35.275743+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368982] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_snaptests}} ----------------------------------------------------------------- time: 00:23:40 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368982/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368982/ sentry: https://sentry.ceph.com/organizations/ceph/?query=e1adc006bba04cce9ec48bb378bc901c "2023-08-15T23:32:16.279157+0000 mgr.x (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368983] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/data-scan} ----------------------------------------------------------------- time: 00:20:12 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368983/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368983/ sentry: https://sentry.ceph.com/organizations/ceph/?query=55cc4ce14a254ab28fda46d1ce1c304f "2023-08-15T23:29:09.561737+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368985] fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/5 tasks/dbench validater/valgrind} ----------------------------------------------------------------- time: 00:35:08 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368985/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368985/ sentry: https://sentry.ceph.com/organizations/ceph/?query=538df1188f2a4e2fa572a25a824e5da2 "2023-08-15T23:42:45.105396+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368986] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/exports} ----------------------------------------------------------------- time: 00:19:31 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368986/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368986/ sentry: https://sentry.ceph.com/organizations/ceph/?query=4609cc43bb7840139e0b53575772b408 "2023-08-15T23:32:28.483086+0000 mgr.y (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log [7368987] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:19:37 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368987/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368987/ sentry: https://sentry.ceph.com/organizations/ceph/?query=bd2173f6877741e693c79e9924fe91aa "2023-08-15T23:34:26.058067+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: devicehealth" in cluster log Dead ================================================================= [7368818] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{yes}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368818/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368818/ Error reimaging machines: Failed to power on smithi188 [7368819] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client-readahead} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368819/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368819/ Error reimaging machines: 'wait for sentinel on smithi181' reached maximum tries (35) after waiting for 1800 seconds [7368820] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_trivial_sync}} ----------------------------------------------------------------- time: 00:03:23 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368820/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368820/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ffd1b45c17fd44eda30ef5517836eaff ['/dev/vg_nvme: already exists in filesystem', "Creating volume group 'vg_nvme' failed", "Run `vgcreate --help' for more information."] [7368821] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-nautilus 1-client 2-upgrade 3-compat_client/pacific}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368821/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368821/ Error reimaging machines: Expected smithi160's OS to be centos 8 but found ubuntu 22.04 [7368822] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client-recovery} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368822/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368822/ Error reimaging machines: Expected smithi149's OS to be ubuntu 20.04 but found centos 9 [7368823] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368823/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368823/ Error reimaging machines: Expected smithi033's OS to be rhel 8.4 but found rhel 8.6 [7368825] fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368825/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368825/ Error reimaging machines: Expected smithi086's OS to be ubuntu 20.04 but found rhel 8.6 [7368826] fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{rhel_8} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/libcephfs/{frag test}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368826/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368826/ Error reimaging machines: 500 Server Error: Internal Server Error for url: http://fog.front.sepia.ceph.com/fog/host/191/task [7368827] fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_dbench_iozone} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368827/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368827/ Error reimaging machines: Expected smithi086's OS to be rhel 8.4 but found rhel 8.6 [7368828] fs/multiclient/{begin clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cephfs_misc_tests} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368828/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368828/ Error reimaging machines: Expected smithi099's OS to be ubuntu 20.04 but found ubuntu 22.04 [7368830] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse ms_mode/{crc} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsync-tester} wsync/{yes}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368830/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368830/ Error reimaging machines: Expected smithi170's OS to be centos 8 but found ubuntu 22.04 [7368831] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/ffsb}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368831/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368831/ Error reimaging machines: Failed to power on smithi052 [7368832] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/data-scan} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368832/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368832/ Error reimaging machines: Expected smithi103's OS to be rhel 8.4 but found ubuntu 22.04 [7368833] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368833/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368833/ Error reimaging machines: 'wait for sentinel on smithi151' reached maximum tries (35) after waiting for 1800 seconds [7368834] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/exports} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368834/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368834/ Error reimaging machines: 'wait for sentinel on smithi173' reached maximum tries (35) after waiting for 1800 seconds [7368835] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368835/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368835/ Error reimaging machines: Expected smithi142's OS to be centos 8 but found centos 9 [7368836] fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/clone}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368836/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368836/ Error reimaging machines: Failed to power on smithi195 [7368837] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/forward-scrub} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368837/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368837/ Error reimaging machines: Expected smithi033's OS to be ubuntu 20.04 but found rhel 8.6 [7368838] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{yes}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368838/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368838/ Error reimaging machines: Expected smithi149's OS to be rhel 8.4 but found centos 9 [7368839] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/iozone}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368839/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368839/ Error reimaging machines: Expected smithi105's OS to be centos 8 but found ubuntu 22.04 [7368840] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-nautilus 1-client 2-upgrade 3-compat_client/no}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368840/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368840/ Error reimaging machines: Expected smithi194's OS to be centos 8 but found centos 9 [7368847] fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368847/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368847/ Error reimaging machines: 500 Server Error: Internal Server Error for url: http://fog.front.sepia.ceph.com/fog/host/144/task [7368848] fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_ffsb traceless/50pc} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368848/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368848/ Error reimaging machines: Expected smithi039's OS to be ubuntu 20.04 but found rhel 8.4 [7368849] fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/3 tasks/dbench validater/valgrind} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368849/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368849/ Error reimaging machines: Expected smithi149's OS to be ubuntu 20.04 but found centos 9 [7368850] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/iogen} wsync/{no}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368850/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368850/ Error reimaging machines: Expected smithi167's OS to be centos 8 but found ubuntu 20.04 [7368851] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_snaptests}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368851/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368851/ Error reimaging machines: Expected smithi067's OS to be ubuntu 20.04 but found centos 9 [7368852] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368852/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368852/ Error reimaging machines: Failed to power on smithi079 [7368853] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mds-flush} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368853/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368853/ Error reimaging machines: Expected smithi102's OS to be centos 8 but found ubuntu 20.04 [7368854] fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_untarbuild_blogbench} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368854/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368854/ Error reimaging machines: Failed to power on smithi110 [7368855] fs/multiclient/{begin clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/ior-shared-file} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368855/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368855/ Error reimaging machines: Expected smithi194's OS to be ubuntu 20.04 but found centos 9 [7368856] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mds-full} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368856/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368856/ Error reimaging machines: Expected smithi039's OS to be ubuntu 20.04 but found rhel 8.4 [7368857] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368857/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368857/ Error reimaging machines: Expected smithi105's OS to be rhel 8.4 but found ubuntu 22.04 [7368858] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mds_creation_retry} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368858/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368858/ Error reimaging machines: Expected smithi131's OS to be rhel 8.4 but found centos 8 [7368859] fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/misc}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368859/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368859/ Error reimaging machines: Expected smithi123's OS to be centos 8 but found rhel 8.6 [7368860] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_fsstress}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368860/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368860/ Error reimaging machines: Expected smithi005's OS to be ubuntu 20.04 but found rhel 8.6 [7368874] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/iozone}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368874/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368874/ Error reimaging machines: Expected smithi131's OS to be centos 8 but found rhel 8.4 [7368877] fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress traceless/50pc} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368877/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368877/ Error reimaging machines: Failed to power on smithi055 [7368879] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/quota} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368879/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368879/ Error reimaging machines: Expected smithi112's OS to be rhel 8.4 but found centos 9 [7368880] fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/5 tasks/fsstress validater/lockdep} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368880/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368880/ Error reimaging machines: Expected smithi084's OS to be centos 8 but found centos 9 [7368881] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_trivial_sync}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368881/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368881/ Error reimaging machines: Expected smithi150's OS to be centos 8 but found rhel 8.6 [7368882] fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_dbench_iozone} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368882/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368882/ Error reimaging machines: Expected smithi119's OS to be ubuntu 20.04 but found ubuntu 22.04 [7368886] fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/snapshot}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368886/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368886/ Error reimaging machines: Failed to power on smithi154 [7368894] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/misc} wsync/{no}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368894/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368894/ Error reimaging machines: Expected smithi154's OS to be centos 8 but found ubuntu 20.04 [7368895] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/snap-schedule} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368895/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368895/ Error reimaging machines: 500 Server Error: Internal Server Error for url: http://fog.front.sepia.ceph.com/fog/host/237/task [7368896] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368896/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368896/ [7368904] fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368904/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368904/ Error reimaging machines: Expected smithi154's OS to be rhel 8.4 but found ubuntu 20.04 [7368911] fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_untarbuild_blogbench} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368911/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368911/ Error reimaging machines: Failed to power on smithi154 [7368923] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/test_journal_migration} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368923/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368923/ Error reimaging machines: Expected smithi154's OS to be rhel 8.4 but found centos 8 [7368924] fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/1 tasks/dbench validater/valgrind} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368924/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368924/ Error reimaging machines: Failed to power on smithi099 [7368927] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368927/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368927/ [7368929] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/workunit/dir-max-entries} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368929/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368929/ Error reimaging machines: Failed to reconnect to smithi132 [7368932] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/dbench} wsync/{yes}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368932/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368932/ Error reimaging machines: 'wait for sentinel on smithi086' reached maximum tries (35) after waiting for 1800 seconds [7368933] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/trivial_sync}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368933/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368933/ Error reimaging machines: Failed to power on smithi103 [7368934] fs/upgrade/nofs/{bluestore-bitmap centos_latest conf/{client mds mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-octopus 1-upgrade}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368934/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368934/ Error reimaging machines: Expected smithi057's OS to be centos 8 but found ubuntu 20.04 [7368935] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/acls} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368935/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368935/ Error reimaging machines: Failed to power on smithi008 [7368938] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368938/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368938/ Error reimaging machines: 'wait for sentinel on smithi143' reached maximum tries (35) after waiting for 1800 seconds [7368939] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/admin} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368939/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368939/ Error reimaging machines: Failed to power on smithi190 [7368940] fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/basic}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368940/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368940/ Error reimaging machines: Expected smithi154's OS to be rhel 8.4 but found centos 8 [7368941] fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_dbench_iozone} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368941/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368941/ Error reimaging machines: Expected smithi175's OS to be centos 8 but found ubuntu 20.04 [7368942] fs/multiclient/{begin clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/ior-shared-file} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368942/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368942/ Error reimaging machines: Expected smithi031's OS to be ubuntu 20.04 but found rhel 8.4 [7368943] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/alternate-pool} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368943/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368943/ Error reimaging machines: Expected smithi008's OS to be ubuntu 20.04 but found centos 8 [7368944] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/iozone}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368944/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368944/ Error reimaging machines: Expected smithi132's OS to be centos 8 but found ubuntu 20.04 [7368945] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} ms_mode/{crc} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{yes}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368945/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368945/ Error reimaging machines: Expected smithi150's OS to be centos 8 but found ubuntu 20.04 [7368946] fs/upgrade/upgraded_client/from_nautilus/{bluestore-bitmap centos_latest clusters/{1-mds-1-client-micro} conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-nautilus 1-client-upgrade 2-client-sanity}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368946/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368946/ Error reimaging machines: Expected smithi057's OS to be centos 8 but found ubuntu 20.04 [7368948] fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368948/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368948/ Error reimaging machines: Failed to power on smithi100 [7368949] fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_dbench traceless/50pc} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368949/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368949/ Error reimaging machines: Expected smithi012's OS to be centos 8 but found centos 9 [7368950] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/asok_dump_tree} ----------------------------------------------------------------- time: 00:03:22 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368950/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368950/ sentry: https://sentry.ceph.com/organizations/ceph/?query=eaebf925f11948f3b83a38f4faeaadf7 ['/dev/vg_nvme: already exists in filesystem', "Creating volume group 'vg_nvme' failed", "Run `vgcreate --help' for more information."] [7368951] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_trivial_sync}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368951/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368951/ Error reimaging machines: Expected smithi044's OS to be ubuntu 20.04 but found rhel 8.4 [7368953] fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/3 tasks/fsstress validater/lockdep} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368953/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368953/ Error reimaging machines: Failed to power on smithi161 [7368954] fs/upgrade/volumes/import-legacy/{bluestore-bitmap clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-nautilus 1-client 2-upgrade 3-verify} ubuntu_18.04} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368954/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368954/ Error reimaging machines: 'wait for sentinel on smithi114' reached maximum tries (35) after waiting for 1800 seconds [7368955] fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368955/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368955/ Error reimaging machines: 'wait for sentinel on smithi070' reached maximum tries (35) after waiting for 1800 seconds [7368956] fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/fs/norstats} wsync/{yes}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368956/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368956/ Error reimaging machines: Expected smithi100's OS to be ubuntu 20.04 but found centos 8 [7368957] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/backtrace} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368957/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368957/ Error reimaging machines: 500 Server Error: Internal Server Error for url: http://fog.front.sepia.ceph.com/fog/host/205/task [7368958] fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368958/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368958/ Error reimaging machines: Expected smithi157's OS to be centos 8 but found ubuntu 20.04 [7368959] fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{ubuntu_latest} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/ino_release_cb} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368959/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368959/ Error reimaging machines: Expected smithi148's OS to be ubuntu 20.04 but found rhel 8.4 [7368960] fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368960/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368960/ Error reimaging machines: Expected smithi057's OS to be centos 8 but found ubuntu 20.04 [7368961] fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/ffsb}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368961/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368961/ Error reimaging machines: Failed to power on smithi100 [7368962] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/cap-flush} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368962/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368962/ Error reimaging machines: Failed to power on smithi154 [7368963] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368963/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368963/ Error reimaging machines: 'wait for sentinel on smithi044' reached maximum tries (35) after waiting for 1800 seconds [7368964] fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client-limits} ----------------------------------------------------------------- time: 00:04:13 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368964/ log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368964/ sentry: https://sentry.ceph.com/organizations/ceph/?query=e7fb8000e8034d9d9573bd48bdfe0e97 ['/dev/vg_nvme: already exists in filesystem', "Creating volume group 'vg_nvme' failed", "Run `vgcreate --help' for more information."] Pass ================================================================= [7368861] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:30:10 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368861/ [7368878] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:31:54 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368878/ [7368897] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:34:29 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368897/ [7368928] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:34:47 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368928/ [7368984] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:31:20 info: http://pulpito.ceph.com/yuriw-2023-08-15_16:56:50-fs-wip-yuri7-testing-2023-08-15-0741-pacific-distro-default-smithi/7368984/