2024-02-28T09:33:08.574 INFO:root:teuthology version: 0.0.1.dev249+gcd45576 2024-02-28T09:33:08.574 INFO:teuthology.results:subset: '743/1024' 2024-02-28T09:33:08.575 INFO:teuthology.results:seed: '8782' 2024-02-28T12:04:03.073 INFO:root:teuthology version: 0.0.1.dev249+gcd45576 2024-02-28T12:04:03.106 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2024-02-28T14:21:14.006 INFO:teuthology.results:Tests finished! gathering results... 2024-02-28T14:21:32.353 INFO:scrape:Found 215 jobs 2024-02-28T14:21:54.581 INFO:scrape:Found 10 distinct failure reasons 2024-02-28T14:21:54.581 INFO:scrape:Failure: 'Filesystem' object has no attribute 'run_ceph_cmd' 2024-02-28T14:21:54.582 INFO:scrape:33 jobs 2024-02-28T14:21:54.582 INFO:scrape:suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'session_timeout}', 'standby-replay'] 2024-02-28T14:21:54.583 INFO:scrape:suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/direct_io}}', '5-workunit/fs/misc}}', '5-workunit/fs/norstats}}', '5-workunit/fs/test_o_trunc}}', '5-workunit/kernel_untar_build}}', '5-workunit/postgres}}', '5-workunit/suites/blogbench}}', '5-workunit/suites/dbench}}', '5-workunit/suites/ffsb}}', '5-workunit/suites/fsstress}}', '5-workunit/suites/fsx}}', '5-workunit/suites/fsync-tester}}', '5-workunit/suites/iogen}}', '5-workunit/suites/iozone}}', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'ranks/1', 'ranks/multi/{balancer/automatic', 'ranks/multi/{balancer/random', 'replication/always}', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}', 'wsync/yes}'] 2024-02-28T14:21:54.583 INFO:scrape: 2024-02-28T14:21:54.583 INFO:scrape:Failure: No module named 'tasks.ceph_fuse' 2024-02-28T14:21:54.583 INFO:scrape:4 jobs: ['7575362', '7575364', '7575358', '7575360'] 2024-02-28T14:21:54.583 INFO:scrape:suites intersection: ['1-ceph', '2-logrotate}', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}'] 2024-02-28T14:21:54.583 INFO:scrape:suites union: ['1-ceph', '2-logrotate}', '2-workunit/cfuse_workunit_suites_pjd}}', '2-workunit/suites/iozone}}', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{ubuntu/{latest', 'distro/{ubuntu_20.04}', 'frag', 'fs/functional/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/verify/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'overrides}}', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/1', 'ranks/3', 'session_timeout', 'session_timeout}', 'subvol_versions/create_subvol_version_v1', 'tasks/dbench', 'tasks/strays}', 'tasks/{1-thrash/mon', 'thrashosds-health}', 'validater/lockdep}'] 2024-02-28T14:21:54.583 INFO:scrape: 2024-02-28T14:21:54.583 INFO:scrape:Failure: Command failed on smithi113 with status 1: "sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v18.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 134c513a-d621-11ee-95c3-87774f69a715 -e sha1=7afccd060f911ee22232d7791b04834d5d886a24 -- bash -c 'ceph orch ps'" 2024-02-28T14:21:54.584 INFO:scrape:2 jobs: ['7575207', '7575279'] 2024-02-28T14:21:54.584 INFO:scrape:suites intersection: ['1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/no', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/reef/{v18.2.0}'] 2024-02-28T14:21:54.584 INFO:scrape:suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/no', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/reef/{v18.2.0}'] 2024-02-28T14:21:54.584 INFO:scrape: 2024-02-28T14:21:54.584 INFO:scrape:Failure: SELinux denials found on ubuntu@smithi090.front.sepia.ceph.com: ['type=AVC msg=audit(1709119574.430:191): avc: denied { checkpoint_restore } for pid=1212 comm="agetty" capability=40 scontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tcontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tclass=capability2 permissive=1'] 2024-02-28T14:21:54.584 INFO:scrape:4 jobs: ['7575329', '7575314', '7575235', '7575201'] 2024-02-28T14:21:54.584 INFO:scrape:suites intersection: ['1-ceph', '2-logrotate}', 'conf/{client', 'mds', 'mon', 'osd}'] 2024-02-28T14:21:54.584 INFO:scrape:suites union: ['1-ceph', '2-logrotate}', '2-workunit/cfuse_workunit_suites_pjd}}', 'bluestore-bitmap', 'centos_latest', 'clusters/1-mds-1-client', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/small-cluster', 'conf/{client', 'distro/{centos_latest}', 'frag', 'fs/bugs/client_trim_caps/{begin/{0-install', 'fs/fscrypt/{begin/{0-install', 'fs/snaps/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'ignorelist_health', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/v1', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'msgr-failures/osd-mds-delay', 'multifs', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'overrides/{distro/testing/k-testing}}', 'overrides/{ignorelist_health', 'pg-warn}', 'session_timeout', 'tasks/fscrypt-ffsb}', 'tasks/trim-i24137}', 'tasks/workunit/snaps}', 'tasks/{1-thrash/mon', 'thrashosds-health}'] 2024-02-28T14:21:54.584 INFO:scrape: 2024-02-28T14:21:54.585 INFO:scrape:Failure: "2024-02-28T11:00:39.480171+0000 mon.a (mon.0) 2154 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log 2024-02-28T14:21:54.585 INFO:scrape:4 jobs: ['7575249', '7575342', '7575198', '7575313'] 2024-02-28T14:21:54.585 INFO:scrape:suites intersection: ['1-ceph', '2-logrotate}'] 2024-02-28T14:21:54.585 INFO:scrape:suites union: ['1-ceph', '2-logrotate}', 'centos_latest', 'clients/mirror', 'cluster/1-node', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'debug', 'distro/{centos_latest}', 'distro/{ubuntu_latest}', 'fs/multifs/{begin/{0-install', 'fs/valgrind/{begin/{0-install', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mirror/{cephfs-mirror/one-per-cluster', 'mon', 'mon-debug}', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/ignorelist_health', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/failover}', 'tasks/mirror}}', 'tasks/volumes/{overrides', 'test/clone}}'] 2024-02-28T14:21:54.585 INFO:scrape: 2024-02-28T14:21:54.585 INFO:scrape:Failure: ['The following modules failed to execute: ansible.legacy.setup'] 2024-02-28T14:21:54.585 INFO:scrape:1 jobs: ['7575292'] 2024-02-28T14:21:54.585 INFO:scrape:suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/yes', '4-flush/no', '5-workunit/fs/test_o_trunc}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/3', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{balancer/random', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}'] 2024-02-28T14:21:54.585 INFO:scrape: 2024-02-28T14:21:54.585 INFO:scrape:Failure: Test failure: test_mount_all_caps_absent (tasks.cephfs.test_multifs_auth.TestClientsWithoutAuth) 2024-02-28T14:21:54.586 INFO:scrape:1 jobs: ['7575204'] 2024-02-28T14:21:54.586 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu_20.04}', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/multifs-auth}'] 2024-02-28T14:21:54.586 INFO:scrape: 2024-02-28T14:21:54.586 INFO:scrape:Failure: Test failure: test_fscrypt_dummy_encryption_with_quick_group (tasks.cephfs.test_fscrypt.TestFscrypt) 2024-02-28T14:21:54.586 INFO:scrape:1 jobs: ['7575237'] 2024-02-28T14:21:54.586 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'bluestore-bitmap', 'clusters/1-mds-1-client', 'conf/{client', 'distro/{centos_latest}', 'fs/fscrypt/{begin/{0-install', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/v1', 'osd}', 'overrides/{distro/testing/k-testing}}', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/fscrypt-common}'] 2024-02-28T14:21:54.586 INFO:scrape: 2024-02-28T14:21:54.586 INFO:scrape:Failure: Config file not found: "/home/teuthworker/src/git.ceph.com_ceph-c_7afccd060f911ee22232d7791b04834d5d886a24/qa/tasks/cephadm.conf". 2024-02-28T14:21:54.586 INFO:scrape:1 jobs: ['7575363'] 2024-02-28T14:21:54.586 INFO:scrape:suites: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/reef/{v18.2.0}'] 2024-02-28T14:21:54.586 INFO:scrape: 2024-02-28T14:21:54.587 INFO:scrape:Failure: Test failure: test_acls (tasks.cephfs.test_acls.TestACLs) 2024-02-28T14:21:54.587 INFO:scrape:1 jobs: ['7575388'] 2024-02-28T14:21:54.587 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v2', 'tasks/acls}'] 2024-02-28T14:21:54.587 INFO:scrape: 2024-02-28T14:21:54.598 INFO:teuthology.results:Sending results to vshankar@redhat.com: Test Run: vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi ================================================================= info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/ logs: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/ failed: 51 dead: 1 running: 0 waiting: 0 queued: 0 passed: 163 Fail ================================================================= [7575175] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/no 5-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:26:00 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575175/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575175/ sentry: https://sentry.ceph.com/organizations/ceph/?query=fad104bb28544b5aaea72d9690c767c5 'Filesystem' object has no attribute 'run_ceph_cmd' [7575179] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/legacy wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/yes 5-workunit/suites/fsx}} ----------------------------------------------------------------- time: 00:25:13 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575179/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575179/ sentry: https://sentry.ceph.com/organizations/ceph/?query=42c8656eda534e83a55a3d77aa381deb 'Filesystem' object has no attribute 'run_ceph_cmd' [7575185] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/5 replication/always} standby-replay tasks/{0-subvolume/{with-quota} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/no 5-workunit/suites/fsync-tester}} ----------------------------------------------------------------- time: 00:26:12 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575185/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575185/ sentry: https://sentry.ceph.com/organizations/ceph/?query=98a25e6d0a824e6abfe5a94861ec70a2 'Filesystem' object has no attribute 'run_ceph_cmd' [7575193] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-comp omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/3 replication/default} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/yes 5-workunit/fs/test_o_trunc}} ----------------------------------------------------------------- time: 00:27:30 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575193/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575193/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ebda51bd1b70414da05536a8deb6d8ac 'Filesystem' object has no attribute 'run_ceph_cmd' [7575198] fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/clone}} ----------------------------------------------------------------- time: 01:05:59 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575198/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575198/ "2024-02-28T10:54:13.212194+0000 mon.a (mon.0) 5117 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY)" in cluster log [7575199] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 standby-replay tasks/{0-subvolume/{with-namespace-isolated} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/no 5-workunit/suites/iogen}} ----------------------------------------------------------------- time: 00:25:44 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575199/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575199/ sentry: https://sentry.ceph.com/organizations/ceph/?query=6d18d110d8ad4ccd8d56a6307277b0ed 'Filesystem' object has no attribute 'run_ceph_cmd' [7575201] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_latest} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_pjd}} ----------------------------------------------------------------- time: 00:21:16 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575201/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575201/ SELinux denials found on ubuntu@smithi006.front.sepia.ceph.com: ['type=AVC msg=audit(1709114091.889:202): avc: denied { checkpoint_restore } for pid=1196 comm="agetty" capability=40 scontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tcontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tclass=capability2 permissive=1'] [7575202] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/legacy wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-quota} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/yes 5-workunit/suites/iozone}} ----------------------------------------------------------------- time: 00:23:11 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575202/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575202/ sentry: https://sentry.ceph.com/organizations/ceph/?query=2ea7776115a74631abb0f33170e9cbd7 'Filesystem' object has no attribute 'run_ceph_cmd' [7575204] fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/multifs-auth} ----------------------------------------------------------------- time: 00:15:52 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575204/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575204/ sentry: https://sentry.ceph.com/organizations/ceph/?query=5f7c202bfd0d494192de8b9b801cc756 Test failure: test_mount_all_caps_absent (tasks.cephfs.test_multifs_auth.TestClientsWithoutAuth) [7575207] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/reef/{v18.2.0} 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} ----------------------------------------------------------------- time: 03:42:09 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575207/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575207/ sentry: https://sentry.ceph.com/organizations/ceph/?query=c0f3da5dbb284fc18854a9f4a1378f77 Command failed on smithi113 with status 1: "sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v18.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 134c513a-d621-11ee-95c3-87774f69a715 -e sha1=7afccd060f911ee22232d7791b04834d5d886a24 -- bash -c 'ceph orch ps'" [7575209] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/no 5-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:26:30 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575209/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575209/ sentry: https://sentry.ceph.com/organizations/ceph/?query=d87381fcc5284d81a5f6bcb517314e97 'Filesystem' object has no attribute 'run_ceph_cmd' [7575217] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-comp omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/5 replication/always} standby-replay tasks/{0-subvolume/{with-namespace-isolated} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/yes 5-workunit/direct_io}} ----------------------------------------------------------------- time: 00:26:20 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575217/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575217/ sentry: https://sentry.ceph.com/organizations/ceph/?query=893e9295a0ce4a839344342585d7a707 'Filesystem' object has no attribute 'run_ceph_cmd' [7575220] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/default} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/no 5-workunit/fs/misc}} ----------------------------------------------------------------- time: 00:26:00 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575220/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575220/ sentry: https://sentry.ceph.com/organizations/ceph/?query=b158c3eeaca5421e852593355d63e1e3 'Filesystem' object has no attribute 'run_ceph_cmd' [7575226] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/legacy wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/yes 5-workunit/kernel_untar_build}} ----------------------------------------------------------------- time: 00:23:25 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575226/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575226/ sentry: https://sentry.ceph.com/organizations/ceph/?query=02f227daaf534325a7da1fb374ec194d 'Filesystem' object has no attribute 'run_ceph_cmd' [7575230] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/no 5-workunit/postgres}} ----------------------------------------------------------------- time: 00:25:20 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575230/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575230/ sentry: https://sentry.ceph.com/organizations/ceph/?query=3c580af2a66b4112bd994ab7d784337d 'Filesystem' object has no attribute 'run_ceph_cmd' [7575235] fs/bugs/client_trim_caps/{begin/{0-install 1-ceph 2-logrotate} centos_latest clusters/small-cluster conf/{client mds mon osd} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/trim-i24137} ----------------------------------------------------------------- time: 00:13:20 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575235/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575235/ SELinux denials found on ubuntu@smithi187.front.sepia.ceph.com: ['type=AVC msg=audit(1709115161.770:201): avc: denied { checkpoint_restore } for pid=1143 comm="agetty" capability=40 scontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tcontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tclass=capability2 permissive=1'] [7575237] fs/fscrypt/{begin/{0-install 1-ceph 2-logrotate} bluestore-bitmap clusters/1-mds-1-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/v1 mount overrides/{distro/testing/k-testing}} overrides/{ignorelist_health ignorelist_health_more ignorelist_wrongly_marked_down pg-warn} tasks/fscrypt-common} ----------------------------------------------------------------- time: 01:30:56 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575237/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575237/ sentry: https://sentry.ceph.com/organizations/ceph/?query=a456c77e3cf24335bef506efe6f2dadf Test failure: test_fscrypt_dummy_encryption_with_quick_group (tasks.cephfs.test_fscrypt.TestFscrypt) [7575249] fs/valgrind/{begin/{0-install 1-ceph 2-logrotate} centos_latest debug mirror/{cephfs-mirror/one-per-cluster clients/mirror cluster/1-node mount/fuse overrides/ignorelist_health tasks/mirror}} ----------------------------------------------------------------- time: 02:27:48 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575249/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575249/ "2024-02-28T11:00:39.480171+0000 mon.a (mon.0) 2154 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log [7575250] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-comp omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/5 replication/always} standby-replay tasks/{0-subvolume/{with-namespace-isolated} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/yes 5-workunit/suites/blogbench}} ----------------------------------------------------------------- time: 00:26:44 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575250/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575250/ sentry: https://sentry.ceph.com/organizations/ceph/?query=93e514fef2444fadbe86c1e0513c4541 'Filesystem' object has no attribute 'run_ceph_cmd' [7575255] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/3 replication/default} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/no 5-workunit/suites/dbench}} ----------------------------------------------------------------- time: 00:26:02 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575255/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575255/ sentry: https://sentry.ceph.com/organizations/ceph/?query=d646dec65285467ab66b8464b83af2b0 'Filesystem' object has no attribute 'run_ceph_cmd' [7575262] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/legacy wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/yes 5-workunit/suites/ffsb}} ----------------------------------------------------------------- time: 00:22:47 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575262/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575262/ sentry: https://sentry.ceph.com/organizations/ceph/?query=dd4f13c13c6547a9b7f1bc8f37a8f7e9 'Filesystem' object has no attribute 'run_ceph_cmd' [7575265] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/no 5-workunit/fs/norstats}} ----------------------------------------------------------------- time: 00:25:14 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575265/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575265/ sentry: https://sentry.ceph.com/organizations/ceph/?query=31e8e0c9c9704a4396676363c0dee926 'Filesystem' object has no attribute 'run_ceph_cmd' [7575272] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-comp omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{with-namespace-isolated} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/yes 5-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:26:45 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575272/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575272/ sentry: https://sentry.ceph.com/organizations/ceph/?query=4dc92aca83eb416085b73c59ff447b5b 'Filesystem' object has no attribute 'run_ceph_cmd' [7575278] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/5 replication/always} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/no 5-workunit/suites/fsx}} ----------------------------------------------------------------- time: 00:25:33 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575278/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575278/ sentry: https://sentry.ceph.com/organizations/ceph/?query=f6e3c5b4dc774fcfa1fbc754a3e61c15 'Filesystem' object has no attribute 'run_ceph_cmd' [7575279] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/reef/{v18.2.0} 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} ----------------------------------------------------------------- time: 02:20:01 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575279/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575279/ sentry: https://sentry.ceph.com/organizations/ceph/?query=c4b96da54a624d0fbbcb29f8d523842a Command failed on smithi136 with status 1: "sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v18.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 553fed80-d627-11ee-95c3-87774f69a715 -e sha1=7afccd060f911ee22232d7791b04834d5d886a24 -- bash -c 'ceph orch ps'" [7575286] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/legacy wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/default} standby-replay tasks/{0-subvolume/{with-quota} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/yes 5-workunit/suites/fsync-tester}} ----------------------------------------------------------------- time: 00:22:31 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575286/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575286/ sentry: https://sentry.ceph.com/organizations/ceph/?query=fb267c73c02d4b778f33e71516f4e054 'Filesystem' object has no attribute 'run_ceph_cmd' [7575297] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-comp omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/yes 5-workunit/suites/iogen}} ----------------------------------------------------------------- time: 00:26:29 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575297/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575297/ sentry: https://sentry.ceph.com/organizations/ceph/?query=482f5e3c1308461ea730030f33b60b95 'Filesystem' object has no attribute 'run_ceph_cmd' [7575303] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/5 replication/always} standby-replay tasks/{0-subvolume/{with-quota} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/no 5-workunit/suites/iozone}} ----------------------------------------------------------------- time: 00:25:41 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575303/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575303/ sentry: https://sentry.ceph.com/organizations/ceph/?query=bc18146e1de34ee595eec627b8a0a4be 'Filesystem' object has no attribute 'run_ceph_cmd' [7575307] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/legacy wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/3 replication/default} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/yes 5-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:23:23 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575307/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575307/ sentry: https://sentry.ceph.com/organizations/ceph/?query=52b46fab6b404ac292eaafd7baddca9c 'Filesystem' object has no attribute 'run_ceph_cmd' [7575313] fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/failover} ----------------------------------------------------------------- time: 00:51:51 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575313/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575313/ "2024-02-28T11:48:06.317788+0000 mon.a (mon.0) 3640 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY)" in cluster log [7575314] fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} ----------------------------------------------------------------- time: 00:42:30 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575314/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575314/ SELinux denials found on ubuntu@smithi089.front.sepia.ceph.com: ['type=AVC msg=audit(1709118848.298:191): avc: denied { checkpoint_restore } for pid=1148 comm="agetty" capability=40 scontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tcontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tclass=capability2 permissive=1'] [7575317] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/no 5-workunit/direct_io}} ----------------------------------------------------------------- time: 00:26:16 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575317/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575317/ sentry: https://sentry.ceph.com/organizations/ceph/?query=0acf0deb7a3347e08017b66c149d9ef2 'Filesystem' object has no attribute 'run_ceph_cmd' [7575322] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-comp omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/yes 5-workunit/fs/misc}} ----------------------------------------------------------------- time: 00:26:36 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575322/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575322/ sentry: https://sentry.ceph.com/organizations/ceph/?query=1f72be0a69604424a4c0e2a47ff45c53 'Filesystem' object has no attribute 'run_ceph_cmd' [7575328] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{with-quota} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/no 5-workunit/kernel_untar_build}} ----------------------------------------------------------------- time: 00:25:42 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575328/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575328/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ee4a197408d44b37ba5d67aa42c5828c 'Filesystem' object has no attribute 'run_ceph_cmd' [7575329] fs/fscrypt/{begin/{0-install 1-ceph 2-logrotate} bluestore-bitmap clusters/1-mds-1-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/v1 mount overrides/{distro/testing/k-testing}} overrides/{ignorelist_health ignorelist_health_more ignorelist_wrongly_marked_down pg-warn} tasks/fscrypt-ffsb} ----------------------------------------------------------------- time: 01:57:21 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575329/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575329/ SELinux denials found on ubuntu@smithi090.front.sepia.ceph.com: ['type=AVC msg=audit(1709119574.430:191): avc: denied { checkpoint_restore } for pid=1212 comm="agetty" capability=40 scontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tcontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tclass=capability2 permissive=1'] [7575332] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/legacy wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/5 replication/always} standby-replay tasks/{0-subvolume/{with-quota} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/yes 5-workunit/postgres}} ----------------------------------------------------------------- time: 00:22:42 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575332/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575332/ sentry: https://sentry.ceph.com/organizations/ceph/?query=c2be8468e66c46b6875d7eba743b23b5 'Filesystem' object has no attribute 'run_ceph_cmd' [7575339] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/default} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/no 5-workunit/suites/blogbench}} ----------------------------------------------------------------- time: 00:25:57 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575339/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575339/ sentry: https://sentry.ceph.com/organizations/ceph/?query=e814eaaf11914a77802540f0b74a6432 'Filesystem' object has no attribute 'run_ceph_cmd' [7575342] fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/clone}} ----------------------------------------------------------------- time: 00:57:54 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575342/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575342/ "2024-02-28T11:55:37.283628+0000 mon.a (mon.0) 1136 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log [7575344] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-comp omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/yes 5-workunit/suites/dbench}} ----------------------------------------------------------------- time: 00:26:25 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575344/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575344/ sentry: https://sentry.ceph.com/organizations/ceph/?query=955fbfb879754887b7592f6dc1edfa6a 'Filesystem' object has no attribute 'run_ceph_cmd' [7575354] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{with-quota} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/no 5-workunit/suites/ffsb}} ----------------------------------------------------------------- time: 00:25:33 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575354/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575354/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ed87cde8ba97476cb3680856508fadf3 'Filesystem' object has no attribute 'run_ceph_cmd' [7575358] fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/3 tasks/dbench validater/lockdep} ----------------------------------------------------------------- time: 00:17:59 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575358/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575358/ sentry: https://sentry.ceph.com/organizations/ceph/?query=b3225abf72ad42e3b2b46414d2d4cb00 No module named 'tasks.ceph_fuse' [7575360] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mon 2-workunit/suites/iozone}} ----------------------------------------------------------------- time: 00:16:06 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575360/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575360/ sentry: https://sentry.ceph.com/organizations/ceph/?query=0f17db359fe84ae5bd565debc7c34797 No module named 'tasks.kclient' [7575361] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/legacy wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/5 replication/always} standby-replay tasks/{0-subvolume/{with-namespace-isolated} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/yes 5-workunit/fs/norstats}} ----------------------------------------------------------------- time: 00:22:49 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575361/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575361/ sentry: https://sentry.ceph.com/organizations/ceph/?query=d3c890eadb0545ff992835bf88c099bc 'Filesystem' object has no attribute 'run_ceph_cmd' [7575362] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/strays} ----------------------------------------------------------------- time: 00:17:02 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575362/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575362/ sentry: https://sentry.ceph.com/organizations/ceph/?query=b07062a683134329a5a1d17c744bdbe9 No module named 'tasks.ceph_fuse' [7575363] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/reef/{v18.2.0} 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} ----------------------------------------------------------------- time: 00:11:42 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575363/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575363/ sentry: https://sentry.ceph.com/organizations/ceph/?query=079a5c402d2d44498c6483f24a1bb627 Config file not found: "/home/teuthworker/src/git.ceph.com_ceph- c_7afccd060f911ee22232d7791b04834d5d886a24/qa/tasks/cephadm.conf". [7575364] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_pjd}} ----------------------------------------------------------------- time: 00:16:07 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575364/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575364/ sentry: https://sentry.ceph.com/organizations/ceph/?query=26e30bf99af84a3eb4824c63943652dc No module named 'tasks.kclient' [7575366] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/3 replication/default} standby-replay tasks/{0-subvolume/{with-quota} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/no 5-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:25:27 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575366/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575366/ sentry: https://sentry.ceph.com/organizations/ceph/?query=0dc47985c04d4d999d3744797675628a 'Filesystem' object has no attribute 'run_ceph_cmd' [7575372] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-comp omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 standby-replay tasks/{0-subvolume/{with-namespace-isolated} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/yes 5-workunit/suites/fsx}} ----------------------------------------------------------------- time: 00:26:02 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575372/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575372/ sentry: https://sentry.ceph.com/organizations/ceph/?query=790258030de24993bf83d391dce6e806 'Filesystem' object has no attribute 'run_ceph_cmd' [7575380] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/automatic export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/yes 3-snaps/no 4-flush/no 5-workunit/suites/fsync-tester}} ----------------------------------------------------------------- time: 00:26:40 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575380/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575380/ sentry: https://sentry.ceph.com/organizations/ceph/?query=942161eba3a44ddc8c21a919fd9e94e6 'Filesystem' object has no attribute 'run_ceph_cmd' [7575384] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/legacy wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/yes 5-workunit/fs/test_o_trunc}} ----------------------------------------------------------------- time: 00:24:25 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575384/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575384/ sentry: https://sentry.ceph.com/organizations/ceph/?query=d0d803d06a2a4a1ab4e0cc4d34f52bf0 'Filesystem' object has no attribute 'run_ceph_cmd' [7575388] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/acls} ----------------------------------------------------------------- time: 00:19:43 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575388/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575388/ sentry: https://sentry.ceph.com/organizations/ceph/?query=4ac0d71fa4da4521b5f849463036a343 Test failure: test_acls (tasks.cephfs.test_acls.TestACLs) Dead ================================================================= [7575292] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-quota} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/no 5-workunit/fs/test_o_trunc}} ----------------------------------------------------------------- time: 00:11:20 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575292/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575292/ sentry: https://sentry.ceph.com/organizations/ceph/?query=21c38bcd455e4c38a7b94d73f45f117a ['The following modules failed to execute: ansible.legacy.setup'] Pass ================================================================= [7575174] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} time: 00:39:16 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575174/ [7575176] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_snaptests}} time: 01:21:34 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575176/ [7575177] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/truncate_delay} time: 00:13:58 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575177/ [7575178] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/quincy}} time: 00:24:19 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575178/ [7575180] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/pjd}} time: 00:18:58 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575180/ [7575181] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/workunit/dir-max-entries} time: 00:17:53 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575181/ [7575182] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/workunit/quota} time: 00:20:52 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575182/ [7575183] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_fsstress}} time: 00:22:44 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575183/ [7575184] fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/1 tasks/dbench validater/lockdep} time: 00:54:05 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575184/ [7575186] fs/upgrade/upgraded_client/{bluestore-bitmap centos_8 clusters/{1-mds-1-client-micro} conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-from/pacific 1-mount/mount/fuse 2-workload/stress_tests/{0-client-upgrade 1-tests/blogbench}}} time: 00:25:32 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575186/ [7575187] fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} time: 00:16:43 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575187/ [7575188] fs/libcephfs/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{centos_8} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/libcephfs_python} time: 00:15:00 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575188/ [7575189] fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_latest} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} time: 00:17:07 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575189/ [7575190] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} time: 00:40:25 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575190/ [7575191] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/acls} time: 00:16:01 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575191/ [7575192] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/fs/snaps}} time: 01:52:52 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575192/ [7575194] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/no}} time: 00:31:06 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575194/ [7575195] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/admin} time: 01:18:54 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575195/ [7575196] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} time: 00:44:09 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575196/ [7575197] fs/fscrypt/{begin/{0-install 1-ceph 2-logrotate} bluestore-bitmap clusters/1-mds-1-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/v1 mount overrides/{distro/testing/k-testing}} overrides/{ignorelist_health ignorelist_health_more ignorelist_wrongly_marked_down pg-warn} tasks/fscrypt-pjd} time: 00:19:28 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575197/ [7575200] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/alternate-pool} time: 00:14:04 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575200/ [7575203] fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/mdtest} time: 00:28:54 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575203/ [7575205] fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} time: 00:54:36 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575205/ [7575206] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/asok_dump_tree} time: 00:24:13 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575206/ [7575208] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} time: 00:24:11 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575208/ [7575210] fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} kclient-overrides/{distro/testing/k-testing ms-die-on-skipped} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_dbench_iozone} time: 00:51:30 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575210/ [7575211] fs/traceless/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress traceless/50pc} time: 00:25:24 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575211/ [7575212] fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} time: 00:24:01 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575212/ [7575213] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/auto-repair} time: 00:17:20 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575213/ [7575214] fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_20.04} mount/fuse objectstore-ec/bluestore-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} time: 00:21:25 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575214/ [7575215] fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_20.04} mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} time: 00:19:50 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575215/ [7575216] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_trivial_sync}} time: 00:18:27 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575216/ [7575218] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/backtrace} time: 00:14:33 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575218/ [7575219] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/fs/trivial_sync}} time: 00:19:11 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575219/ [7575221] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/cap-flush} time: 00:14:36 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575221/ [7575222] fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/finisher_per_module}} time: 00:15:51 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575222/ [7575223] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/ffsb}} time: 00:28:56 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575223/ [7575224] fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/3 tasks/fsstress validater/valgrind} time: 01:34:14 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575224/ [7575225] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/client-limits} time: 00:24:52 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575225/ [7575227] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/quincy}} time: 00:24:34 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575227/ [7575228] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mds 2-workunit/suites/iozone}} time: 00:19:35 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575228/ [7575229] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/client-readahead} time: 00:18:09 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575229/ [7575231] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/pjd}} time: 00:18:03 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575231/ [7575232] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} time: 00:37:53 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575232/ [7575233] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/client-recovery} time: 01:05:44 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575233/ [7575234] fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} time: 00:22:14 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575234/ [7575236] fs/cephadm/multivolume/{0-start 1-mount 2-workload/dbench distro/single-container-host} time: 00:51:22 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575236/ [7575238] fs/full/{begin/{0-install 1-ceph 2-logrotate} clusters/1-node-1-mds-1-osd conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-bitmap overrides overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mgr-osd-full} time: 00:21:55 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575238/ [7575239] fs/libcephfs/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{centos_8} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client} time: 00:13:57 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575239/ [7575240] fs/mirror/{begin/{0-install 1-ceph 2-logrotate} cephfs-mirror/one-per-cluster clients/{mirror} cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health} supported-random-distros$/{centos_8} tasks/mirror} time: 02:12:42 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575240/ [7575241] fs/mirror-ha/{begin/{0-install 1-ceph 2-logrotate} cephfs-mirror/{1-volume-create-rm 2-three-per-cluster} clients/{mirror} cluster/{1-node} objectstore/bluestore-bitmap overrides/{ignorelist_health} supported-random-distro$/{centos_latest} workloads/cephfs-mirror-ha-workunit} time: 00:37:50 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575241/ [7575242] fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cephfs_misc_tests} time: 01:00:56 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575242/ [7575243] fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/failover} time: 00:58:47 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575243/ [7575244] fs/nfs/{cluster/{1-node} overrides/ignorelist_health supported-random-distros$/{ubuntu_20.04} tasks/nfs} time: 00:45:12 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575244/ [7575245] fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} time: 00:18:25 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575245/ [7575246] fs/shell/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/cephfs-shell} time: 00:49:51 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575246/ [7575247] fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} time: 01:04:08 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575247/ [7575248] fs/top/{begin/{0-install 1-ceph 2-logrotate} cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/ignorelist_health supported-random-distros$/{centos_8} tasks/fstop} time: 00:15:22 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575248/ [7575251] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/no}} time: 00:31:07 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575251/ [7575252] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/none objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/iozone}} time: 00:24:28 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575252/ [7575253] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/damage} time: 00:26:59 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575253/ [7575254] fs/upgrade/nofs/{bluestore-bitmap centos_8 conf/{client mds mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-pacific 1-upgrade}} time: 00:13:13 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575254/ [7575256] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/fs/snaps}} time: 01:30:32 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575256/ [7575257] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/data-scan} time: 00:46:24 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575257/ [7575258] fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} kclient-overrides/{distro/testing/k-testing ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_dbench_iozone} time: 00:45:38 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575258/ [7575259] fs/traceless/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_20.04} mount/fuse objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_blogbench traceless/50pc} time: 00:24:58 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575259/ [7575260] fs/upgrade/upgraded_client/{bluestore-bitmap centos_8 clusters/{1-mds-1-client-micro} conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-from/pacific 1-mount/mount/fuse 2-workload/new_ops/{0-clients/fuse-upgrade 1-client-sanity}}} time: 00:14:25 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575260/ [7575261] fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/misc}} time: 00:22:03 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575261/ [7575263] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_snaptests}} time: 01:24:19 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575263/ [7575264] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/exports} time: 00:53:51 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575264/ [7575266] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} time: 00:40:06 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575266/ [7575267] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/osd 2-workunit/suites/ffsb}} time: 00:40:06 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575267/ [7575268] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/forward-scrub} time: 00:24:48 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575268/ [7575269] fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} time: 00:23:02 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575269/ [7575270] fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_20.04} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} time: 00:18:53 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575270/ [7575271] fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} time: 00:21:47 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575271/ [7575273] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_fsstress}} time: 00:27:06 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575273/ [7575274] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/fragment} time: 00:26:24 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575274/ [7575275] fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/5 tasks/dbench validater/lockdep} time: 00:47:57 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575275/ [7575276] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} time: 00:21:22 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575276/ [7575277] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/journal-repair} time: 00:28:08 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575277/ [7575280] fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/ior-shared-file} time: 00:16:02 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575280/ [7575281] fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/multifs-auth} time: 00:29:04 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575281/ [7575282] fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} time: 00:55:37 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575282/ [7575283] fs/upgrade/upgraded_client/{bluestore-bitmap centos_8 clusters/{1-mds-1-client-micro} conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-from/nautilus 1-mount/mount/fuse 2-workload/stress_tests/{0-client-upgrade 1-tests/fsstress}}} time: 00:20:08 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575283/ [7575284] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/mds-flush} time: 00:15:51 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575284/ [7575285] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_pjd}} time: 00:18:15 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575285/ [7575287] fs/fscrypt/{begin/{0-install 1-ceph 2-logrotate} bluestore-bitmap clusters/1-mds-1-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/v1 mount overrides/{distro/testing/k-testing}} overrides/{ignorelist_health ignorelist_health_more ignorelist_wrongly_marked_down pg-warn} tasks/fscrypt-dbench} time: 01:11:45 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575287/ [7575288] fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/snapshot}} time: 00:43:23 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575288/ [7575289] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/quincy}} time: 00:23:52 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575289/ [7575290] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/mds-full} time: 00:25:19 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575290/ [7575291] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/fs/trivial_sync}} time: 00:17:26 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575291/ [7575293] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/mds_creation_retry} time: 00:13:21 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575293/ [7575294] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/osd 2-workunit/suites/iozone}} time: 00:20:04 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575294/ [7575295] fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_untarbuild_blogbench} time: 00:42:39 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575295/ [7575296] fs/traceless/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_dbench traceless/50pc} time: 00:50:22 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575296/ [7575298] fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} time: 00:17:58 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575298/ [7575299] fs/libcephfs/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{centos_8} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/ino_release_cb} time: 00:13:49 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575299/ [7575300] fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} time: 00:22:36 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575300/ [7575301] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/metrics} time: 00:37:34 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575301/ [7575302] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_trivial_sync}} time: 00:16:32 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575302/ [7575304] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/no}} time: 00:32:37 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575304/ [7575305] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/multimds_misc} time: 00:24:47 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575305/ [7575306] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/pjd}} time: 00:18:43 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575306/ [7575308] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} time: 00:39:50 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575308/ [7575309] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/openfiletable} time: 00:16:12 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575309/ [7575310] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/ffsb}} time: 00:27:59 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575310/ [7575311] fs/upgrade/upgraded_client/{bluestore-bitmap centos_8 clusters/{1-mds-1-client-micro} conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-from/pacific 1-mount/mount/fuse 2-workload/stress_tests/{0-client-upgrade 1-tests/kernel_untar_build}}} time: 01:34:41 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575311/ [7575312] fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/mdtest} time: 00:17:05 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575312/ [7575315] fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/1 tasks/fsstress validater/valgrind} time: 01:29:14 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575315/ [7575316] fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/basic}} time: 01:51:13 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575316/ [7575318] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/pool-perm} time: 00:17:02 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575318/ [7575319] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/fs/snaps}} time: 01:49:17 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575319/ [7575320] fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} time: 00:23:39 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575320/ [7575321] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/quota} time: 00:16:55 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575321/ [7575323] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/iozone}} time: 00:22:49 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575323/ [7575324] fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_20.04} mount/fuse objectstore-ec/bluestore-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} time: 00:18:59 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575324/ [7575325] fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_20.04} mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} time: 00:19:24 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575325/ [7575326] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/recovery-fs} time: 00:12:17 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575326/ [7575327] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} time: 00:40:32 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575327/ [7575330] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mon 2-workunit/suites/ffsb}} time: 00:33:10 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575330/ [7575331] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/scrub} time: 00:31:13 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575331/ [7575333] fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} kclient-overrides/{distro/testing/k-testing ms-die-on-skipped} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_dbench_iozone} time: 00:46:49 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575333/ [7575334] fs/traceless/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_ffsb traceless/50pc} time: 00:39:01 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575334/ [7575335] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_snaptests}} time: 01:18:11 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575335/ [7575336] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/quincy}} time: 00:23:37 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575336/ [7575337] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/sessionmap} time: 00:16:46 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575337/ [7575338] fs/upgrade/upgraded_client/{bluestore-bitmap centos_8 clusters/{1-mds-1-client-micro} conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-from/nautilus 1-mount/mount/fuse 2-workload/stress_tests/{0-client-upgrade 1-tests/dbench}}} time: 00:55:37 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575338/ [7575340] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} time: 00:27:00 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575340/ [7575341] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/snap-schedule} time: 02:18:24 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575341/ [7575343] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} time: 00:40:17 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575343/ [7575345] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/fs/trivial_sync}} time: 00:18:16 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575345/ [7575346] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/snap_schedule_snapdir} time: 00:15:51 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575346/ [7575347] fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} time: 00:16:20 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575347/ [7575348] fs/cephadm/renamevolume/{0-start 1-rename distro/single-container-host overrides/ignorelist_health} time: 00:23:15 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575348/ [7575349] fs/libcephfs/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{rhel_8} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/libcephfs/{frag test}} time: 00:31:59 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575349/ [7575350] fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cephfs_misc_tests} time: 01:01:11 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575350/ [7575351] fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/multifs-auth} time: 00:38:51 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575351/ [7575352] fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} time: 00:16:21 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575352/ [7575353] fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} time: 01:35:37 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575353/ [7575355] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_fsstress}} time: 00:26:35 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575355/ [7575356] fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/no}} time: 00:31:14 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575356/ [7575357] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/snapshots} time: 00:28:54 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575357/ [7575359] fs/upgrade/nofs/{bluestore-bitmap centos_8 conf/{client mds mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-pacific 1-upgrade}} time: 00:13:18 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575359/ [7575365] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/test_journal_migration} time: 00:23:54 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575365/ [7575367] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/pjd}} time: 00:22:28 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575367/ [7575368] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/truncate_delay} time: 00:13:46 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575368/ [7575369] fs/upgrade/upgraded_client/{bluestore-bitmap centos_8 clusters/{1-mds-1-client-micro} conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-from/pacific 1-mount/mount/fuse 2-workload/stress_tests/{0-client-upgrade 1-tests/iozone}}} time: 00:18:55 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575369/ [7575370] fs/fscrypt/{begin/{0-install 1-ceph 2-logrotate} bluestore-bitmap clusters/1-mds-1-client conf/{client mds mon osd} distro/{centos_latest} mount/kclient/{mount-syntax/v1 mount overrides/{distro/testing/k-testing}} overrides/{ignorelist_health ignorelist_health_more ignorelist_wrongly_marked_down pg-warn} tasks/fscrypt-iozone} time: 02:20:11 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575370/ [7575371] fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_20.04} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/finisher_per_module}} time: 00:15:07 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575371/ [7575373] fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_untarbuild_blogbench} time: 00:41:54 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575373/ [7575374] fs/traceless/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress traceless/50pc} time: 00:20:09 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575374/ [7575375] fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_8 clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} time: 00:22:42 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575375/ [7575376] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v2 tasks/workunit/dir-max-entries} time: 00:16:17 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575376/ [7575377] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_trivial_sync}} time: 00:16:40 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575377/ [7575378] fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} time: 00:19:12 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575378/ [7575379] fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_latest} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} time: 00:17:39 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575379/ [7575381] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} time: 00:39:36 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575381/ [7575382] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} subvol_versions/create_subvol_version_v1 tasks/workunit/quota} time: 00:16:58 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575382/ [7575383] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{client-shutdown frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/fs/snaps}} time: 01:58:30 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575383/ [7575385] fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/ior-shared-file} time: 00:14:23 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575385/ [7575386] fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/failover} time: 00:54:27 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575386/ [7575387] fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} time: 00:41:35 info: https://pulpito.ceph.com/vshankar-2024-02-28_09:32:44-fs-wip-vshankar-testing1-reef-2024-02-28-1133-testing-default-smithi/7575387/