2022-04-08T04:59:07.743 INFO:root:teuthology version: 1.1.0-a347b2ac 2022-04-08T04:59:07.746 INFO:teuthology.results:subset: '21/32' 2022-04-08T04:59:07.746 INFO:teuthology.results:seed: '6329' 2022-04-08T06:26:25.655 INFO:root:teuthology version: 1.1.0-a347b2ac 2022-04-08T06:26:25.694 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2022-04-08T12:53:54.785 INFO:teuthology.results:Tests finished! gathering results... 2022-04-08T12:53:59.548 INFO:scrape:Found 50 jobs 2022-04-08T12:54:06.350 INFO:scrape:Found 9 distinct failure reasons 2022-04-08T12:54:06.351 INFO:scrape:Timeout 3h running clone.client.0/qa/workunits/suites/ffsb.sh 2022-04-08T12:54:06.352 INFO:scrape:1 jobs: ['6782344'] 2022-04-08T12:54:06.352 INFO:scrape:suites: ['1-cephadm', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs/workload/{0-rhel_8', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/{crc}', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/1', 'scrub/yes', 'session_timeout', 'standby-replay', 'tasks/{0-check-counter', 'whitelist_health', 'whitelist_wrongly_marked_down}', 'workunit/suites/ffsb}', 'wsync/{yes}}'] 2022-04-08T12:54:06.353 INFO:scrape: 2022-04-08T12:54:06.353 INFO:scrape:Failure: "2022-04-08T06:58:36.607199+0000 mds.e (mds.0) 23 : cluster [WRN] client.4604 isn't responding to mclientcaps(revoke), ino 0x10000004ae2 pending pFc issued pFcb, sent 304.996827 seconds ago" in cluster log 2022-04-08T12:54:06.354 INFO:scrape:2 jobs: ['6782369', '6782332'] 2022-04-08T12:54:06.354 INFO:scrape:suites intersection: ['1-ceph', '2-logrotate}', '2-workunit/fs/snaps}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'fs/thrash/workloads/{begin/{0-install', 'mds', 'mon', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{frag', 'prefetch_entire_dirfrags/no', 'races', 'session_timeout', 'tasks/{1-thrash/osd', 'thrashosds-health', 'whitelist_health', 'whitelist_wrongly_marked_down}'] 2022-04-08T12:54:06.355 INFO:scrape:suites union: ['1-ceph', '2-logrotate}', '2-workunit/fs/snaps}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'fs/thrash/workloads/{begin/{0-install', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'races', 'ranks/1', 'ranks/3', 'session_timeout', 'tasks/{1-thrash/osd', 'thrashosds-health', 'whitelist_health', 'whitelist_wrongly_marked_down}'] 2022-04-08T12:54:06.355 INFO:scrape: 2022-04-08T12:54:06.356 INFO:scrape:Dead: b'2022-04-08T12:08:53.952 DEBUG:teuthology.exit:Finished running handlers' 2022-04-08T12:54:06.356 INFO:scrape:4 jobs: ['6782347', '6782364', '6782366', '6782331'] 2022-04-08T12:54:06.356 INFO:scrape:suites intersection: ['1-volume/{0-create', '2-client', '3-inline/yes', '3-upgrade-with-workload', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'mds', 'mon', 'osd}', 'overrides/{pg-warn', 'roles', 'syntax', 'whitelist_health', 'whitelist_wrongly_marked_down}'] 2022-04-08T12:54:06.357 INFO:scrape:suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client', '3-inline/yes', '3-upgrade-with-workload', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'mds', 'mon', 'osd}', 'overrides/{pg-warn', 'roles', 'syntax', 'tasks/{0-from/pacific', 'tasks/{0-from/v16.2.4', 'whitelist_health', 'whitelist_wrongly_marked_down}'] 2022-04-08T12:54:06.357 INFO:scrape: 2022-04-08T12:54:06.358 INFO:scrape:Failure: error during scrub thrashing: reached maximum tries (30) after waiting for 900 seconds 2022-04-08T12:54:06.358 INFO:scrape:3 jobs: ['6782368', '6782353', '6782358'] 2022-04-08T12:54:06.359 INFO:scrape:suites intersection: ['2-logrotate}', 'conf/{client', 'mds', 'mon', 'osd}', 'overrides/{frag', 'session_timeout', 'whitelist_health', 'whitelist_wrongly_marked_down}'] 2022-04-08T12:54:06.359 INFO:scrape:suites union: ['1-ceph', '1-cephadm', '2-logrotate}', '2-workunit/cfuse_workunit_snaptests}}', '2-workunit/fs/snaps}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-2c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/workload/{0-rhel_8', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/{secure}', 'msgr-failures/osd-mds-delay', 'multifs', 'objectstore-ec/bluestore-bitmap', 'objectstore/bluestore-bitmap', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/no', 'races', 'ranks/1', 'ranks/5', 'scrub/yes', 'session_timeout', 'standby-replay', 'tasks/{0-check-counter', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'thrashosds-health', 'whitelist_health', 'whitelist_wrongly_marked_down}', 'workunit/fs/misc}', 'wsync/{no}}'] 2022-04-08T12:54:06.360 INFO:scrape: 2022-04-08T12:54:06.360 INFO:scrape:Failure: Test failure: test_acls (tasks.cephfs.test_acls.TestACLs) 2022-04-08T12:54:06.360 INFO:scrape:1 jobs: ['6782352'] 2022-04-08T12:54:06.361 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{no_client_pidfile', 'rhel_8}', 'tasks/acls}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] 2022-04-08T12:54:06.361 INFO:scrape: 2022-04-08T12:54:06.362 INFO:scrape:Timeout 3h running clone.client.0/qa/workunits/fs/snaps/untar_snap_rm.sh 2022-04-08T12:54:06.362 INFO:scrape:2 jobs: ['6782349', '6782341'] 2022-04-08T12:54:06.363 INFO:scrape:suites intersection: ['1-ceph', '2-logrotate}', '2-workunit/fs/snaps}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'fs/thrash/workloads/{begin/{0-install', 'mds', 'mon', 'mount/fuse', 'msgr-failures/osd-mds-delay', 'osd}', 'overrides/{frag', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/3', 'session_timeout', 'tasks/{1-thrash/osd', 'thrashosds-health', 'whitelist_health', 'whitelist_wrongly_marked_down}'] 2022-04-08T12:54:06.363 INFO:scrape:suites union: ['1-ceph', '2-logrotate}', '2-workunit/fs/snaps}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'fs/thrash/workloads/{begin/{0-install', 'mds', 'mon', 'mount/fuse', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{frag', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/3', 'session_timeout', 'tasks/{1-thrash/osd', 'thrashosds-health', 'whitelist_health', 'whitelist_wrongly_marked_down}'] 2022-04-08T12:54:06.364 INFO:scrape: 2022-04-08T12:54:06.364 INFO:scrape:Failure: The following counters failed to be set on mds daemons: {'mds.imported', 'mds.exported'} 2022-04-08T12:54:06.364 INFO:scrape:1 jobs: ['6782335'] 2022-04-08T12:54:06.365 INFO:scrape:suites: ['1-cephadm', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs/workload/{0-rhel_8', 'mds', 'mon', 'mount/fuse', 'ms_mode/{crc}', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{frag', 'ranks/3', 'scrub/yes', 'session_timeout', 'standby-replay', 'tasks/{0-check-counter', 'whitelist_health', 'whitelist_wrongly_marked_down}', 'workunit/suites/iogen}', 'wsync/{yes}}'] 2022-04-08T12:54:06.365 INFO:scrape: 2022-04-08T12:54:06.366 INFO:scrape:Timeout 3h running clone.client.1/qa/workunits/suites/fsstress.sh 2022-04-08T12:54:06.366 INFO:scrape:1 jobs: ['6782372'] 2022-04-08T12:54:06.367 INFO:scrape:suites: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '3-inline/yes', '3-upgrade-with-workload', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'mds', 'mon', 'osd}', 'overrides/{pg-warn', 'roles', 'syntax', 'tasks/{0-from/v16.2.4', 'whitelist_health', 'whitelist_wrongly_marked_down}'] 2022-04-08T12:54:06.367 INFO:scrape: 2022-04-08T12:54:06.367 INFO:scrape:Failure: Command failed (workunit test suites/fsstress.sh) on smithi083 with status 126: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=8b9f952f608a1f0a8229c7a4fb9b2554d98f1c99 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh' 2022-04-08T12:54:06.368 INFO:scrape:1 jobs: ['6782357'] 2022-04-08T12:54:06.368 INFO:scrape:suites: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client', '3-inline/yes', '3-upgrade-with-workload', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'mds', 'mon', 'osd}', 'overrides/{pg-warn', 'roles', 'syntax', 'tasks/{0-from/pacific', 'whitelist_health', 'whitelist_wrongly_marked_down}'] 2022-04-08T12:54:06.369 INFO:scrape: 2022-04-08T12:54:06.371 INFO:teuthology.results:Sending results to vshankar@redhat.com: Test Run: vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi ================================================================= info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/ logs: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/ failed: 11 dead: 5 running: 0 waiting: 0 queued: 0 passed: 34 Fail ================================================================= [6782332] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 03:49:06 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782332/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782332/ "2022-04-08T05:52:16.303033+0000 mds.e (mds.0) 19 : cluster [WRN] client.4803 isn't responding to mclientcaps(revoke), ino 0x1000000141c pending pAsLsXsFscr issued pAsLsXsFsxcrwb, sent 300.008890 seconds ago" in cluster log [6782335] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{crc} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/iogen} wsync/{yes}} ----------------------------------------------------------------- time: 00:54:40 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782335/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782335/ The following counters failed to be set on mds daemons: {'mds.imported', 'mds.exported'} [6782341] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 06:13:11 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782341/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782341/ sentry: https://sentry.ceph.com/organizations/ceph/?query=07e79de73b794394b18ed386cd71e2ed Command failed (workunit test fs/snaps/untar_snap_rm.sh) on smithi067 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=8b9f952f608a1f0a8229c7a4fb9b2554d98f1c99 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clo ne.client.0/qa/workunits/fs/snaps/untar_snap_rm.sh' [6782344] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{crc} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{yes}} ----------------------------------------------------------------- time: 03:29:20 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782344/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782344/ sentry: https://sentry.ceph.com/organizations/ceph/?query=8f3e549118f5445c8bc314f8aee8acc1 Command failed (workunit test suites/ffsb.sh) on smithi036 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=8b9f952f608a1f0a8229c7a4fb9b2554d98f1c99 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh' [6782349] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 04:47:42 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782349/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782349/ sentry: https://sentry.ceph.com/organizations/ceph/?query=09f6bb047a804f9abbc668dc1f84c5bf Command failed (workunit test fs/snaps/untar_snap_rm.sh) on smithi093 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=8b9f952f608a1f0a8229c7a4fb9b2554d98f1c99 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clo ne.client.0/qa/workunits/fs/snaps/untar_snap_rm.sh' [6782352] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/acls} ----------------------------------------------------------------- time: 00:13:13 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782352/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782352/ sentry: https://sentry.ceph.com/organizations/ceph/?query=561fb53e9d1649f6bb5b53a2989ab4fe Test failure: test_acls (tasks.cephfs.test_acls.TestACLs) [6782353] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_snaptests}} ----------------------------------------------------------------- time: 00:22:36 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782353/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782353/ sentry: https://sentry.ceph.com/organizations/ceph/?query=0a8154027a60482e9c75f85449eac58b reached maximum tries (90) after waiting for 540 seconds [6782357] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn syntax whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} ----------------------------------------------------------------- time: 00:19:32 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782357/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782357/ sentry: https://sentry.ceph.com/organizations/ceph/?query=aa592368cbc74c9a969ad31819b8ec8f Command failed (workunit test suites/fsstress.sh) on smithi083 with status 126: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=8b9f952f608a1f0a8229c7a4fb9b2554d98f1c99 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh' [6782358] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{frag prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 00:21:50 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782358/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782358/ sentry: https://sentry.ceph.com/organizations/ceph/?query=e088cbf7b25649839cd21a52f8c8d5df reached maximum tries (90) after waiting for 540 seconds [6782368] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{secure} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/misc} wsync/{no}} ----------------------------------------------------------------- time: 02:12:16 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782368/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782368/ error during scrub thrashing: reached maximum tries (30) after waiting for 900 seconds [6782369] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 04:07:07 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782369/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782369/ "2022-04-08T06:58:36.607199+0000 mds.e (mds.0) 23 : cluster [WRN] client.4604 isn't responding to mclientcaps(revoke), ino 0x10000004ae2 pending pFc issued pFcb, sent 304.996827 seconds ago" in cluster log Dead ================================================================= [6782331] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn syntax whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782331/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782331/ hit max job timeout [6782347] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn syntax whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782347/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782347/ hit max job timeout [6782364] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn syntax whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782364/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782364/ hit max job timeout [6782366] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn syntax whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782366/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782366/ hit max job timeout [6782372] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn syntax whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} ----------------------------------------------------------------- time: 03:34:12 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782372/ log: http://qa-proxy.ceph.com/teuthology/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782372/ sentry: https://sentry.ceph.com/organizations/ceph/?query=4428d4f296f44ce9a16d1a07132dfb16 Command failed (workunit test suites/fsstress.sh) on smithi146 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=8b9f952f608a1f0a8229c7a4fb9b2554d98f1c99 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsstress.sh' Pass ================================================================= [6782326] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/dbench} wsync/{yes}} time: 01:32:30 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782326/ [6782327] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{secure} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{yes}} time: 01:06:11 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782327/ [6782328] fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/failover} time: 00:51:59 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782328/ [6782329] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsstress} wsync/{no}} time: 00:28:51 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782329/ [6782330] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{legacy} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{no}} time: 00:47:26 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782330/ [6782333] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{no}} time: 00:39:10 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782333/ [6782334] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} time: 00:20:41 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782334/ [6782336] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/fs/snaps}} time: 02:04:17 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782336/ [6782337] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/pjd} wsync/{no}} time: 00:28:10 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782337/ [6782338] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{crc} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/direct_io} wsync/{yes}} time: 00:33:49 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782338/ [6782339] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/fs/snaps}} time: 02:37:41 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782339/ [6782340] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/kernel_untar_build} wsync/{yes}} time: 00:53:04 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782340/ [6782342] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{legacy} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/blogbench} wsync/{yes}} time: 00:43:22 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782342/ [6782343] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} time: 00:20:09 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782343/ [6782345] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{crc} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/fs/norstats} wsync/{no}} time: 00:32:18 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782345/ [6782346] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{no}} time: 00:36:51 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782346/ [6782348] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{secure} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsync-tester} wsync/{no}} time: 00:30:30 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782348/ [6782350] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/iogen} wsync/{no}} time: 00:37:13 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782350/ [6782351] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{legacy} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/iozone} wsync/{no}} time: 00:32:06 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782351/ [6782354] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/misc} wsync/{no}} time: 01:04:47 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782354/ [6782355] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{crc} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/kernel_untar_build} wsync/{yes}} time: 02:17:06 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782355/ [6782356] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/dbench} wsync/{yes}} time: 01:04:31 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782356/ [6782359] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{secure} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{yes}} time: 01:07:04 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782359/ [6782360] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{crc} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsstress} wsync/{yes}} time: 00:31:37 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782360/ [6782361] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{secure} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{no}} time: 00:36:40 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782361/ [6782362] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn syntax whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:35:13 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782362/ [6782363] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{no}} time: 00:35:45 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782363/ [6782365] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/metrics} time: 00:24:16 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782365/ [6782367] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{frag prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/iozone}} time: 00:18:48 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782367/ [6782370] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{crc} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/blogbench} wsync/{no}} time: 00:48:01 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782370/ [6782371] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/iozone}} time: 00:16:45 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782371/ [6782373] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsync-tester} wsync/{yes}} time: 00:27:03 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782373/ [6782374] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/snaps}} time: 01:58:16 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782374/ [6782375] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/snap-schedule} time: 00:46:26 info: http://pulpito.ceph.com/vshankar-2022-04-08_04:55:29-fs-wip-vshankar-testing-55110-20220407-173953-testing-default-smithi/6782375/