2022-07-03T15:13:50.927 INFO:root:teuthology version: 1.1.1.dev379+gf58709e3 2022-07-03T15:13:50.929 INFO:teuthology.results:subset: '1/32' 2022-07-03T15:13:50.929 INFO:teuthology.results:seed: '3567' 2022-07-03T20:36:20.805 INFO:root:teuthology version: 1.1.1.dev379+gf58709e3 2022-07-03T20:36:20.836 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2022-07-04T08:36:21.952 WARNING:teuthology.results:test(s) did not finish before timeout of 43200 seconds 2022-07-04T08:36:33.407 INFO:scrape:Found 121 jobs 2022-07-04T08:37:19.489 INFO:scrape:Found 20 distinct failure reasons 2022-07-04T08:37:19.490 INFO:scrape:Failure: SSH connection to smithi036 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' 2022-07-04T08:37:19.491 INFO:scrape:27 jobs: ['6911988', '6911874', '6911978', '6911958', '6911986', '6911879', '6911979', '6911963', '6911984', '6911987', '6911983', '6911909', '6911883', '6911923', '6911946', '6911935', '6911964', '6911900', '6911956', '6911922', '6911888', '6911899', '6911954', '6911938', '6911911', '6911971', '6911977'] 2022-07-04T08:37:19.492 INFO:scrape:suites intersection: ['2-logrotate}', 'conf/{client', 'mds', 'mon', 'mount', 'osd}'] 2022-07-04T08:37:19.493 INFO:scrape:suites union: ['1-ceph', '1-cephadm', '2-logrotate}', '2-workunit/iozone}}', '2-workunit/suites/fsstress}}', '2-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'distro/{ubuntu/{latest', 'distro/{ubuntu_latest}', 'fs/snaps/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/verify/{begin/{0-install', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'mount/kclient/{k-testing', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/secure', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'overrides/{ignorelist_health', 'overrides}}', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/1', 'ranks/3', 'ranks/5', 'scrub/yes', 'session_timeout', 'session_timeout}', 'standby-replay', 'subvolume/{with-namespace-isolated-and-quota}', 'subvolume/{with-namespace-isolated}', 'subvolume/{with-no-extra-options}', 'tasks/fsstress', 'tasks/workunit/snaps}', 'tasks/{0-check-counter', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'thrashosds-health}', 'validater/valgrind}', 'workunit/suites/blogbench}}', 'workunit/suites/ffsb}}', 'workunit/suites/pjd}}', 'wsync/yes}'] 2022-07-04T08:37:19.493 INFO:scrape: 2022-07-04T08:37:19.493 INFO:scrape:Dead: b'2022-07-04T03:22:24.736 DEBUG:teuthology.exit:Finished running handlers' 2022-07-04T08:37:19.494 INFO:scrape:33 jobs 2022-07-04T08:37:19.495 INFO:scrape:suites intersection: ['2-logrotate}', 'conf/{client', 'mds', 'mon', 'osd}'] 2022-07-04T08:37:19.496 INFO:scrape:suites union: ['1-ceph', '1-cephadm', '2-logrotate}', '2-workunit/cfuse_workunit_snaptests}}', '2-workunit/suites/fsstress}}', '2-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'fs/mixed-clients/{begin/{0-install', 'fs/snaps/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/secure', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd-asserts}', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'overrides/{ignorelist_health', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/1', 'ranks/3', 'ranks/5', 'scrub/no', 'scrub/yes', 'session_timeout', 'session_timeout}', 'standby-replay', 'subvolume/{no-subvolume}', 'subvolume/{with-namespace-isolated-and-quota}', 'subvolume/{with-namespace-isolated}', 'subvolume/{with-no-extra-options}', 'subvolume/{with-quota}', 'tasks/asok_dump_tree}', 'tasks/kernel_cfuse_workunits_dbench_iozone}', 'tasks/workunit/snaps}', 'tasks/{0-check-counter', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'thrashosds-health}', 'workunit/direct_io}}', 'workunit/kernel_untar_build}}', 'workunit/suites/blogbench}}', 'workunit/suites/dbench}}', 'workunit/suites/ffsb}}', 'workunit/suites/fsstress}}', 'workunit/suites/fsx}}', 'workunit/suites/pjd}}', 'wsync/yes}'] 2022-07-04T08:37:19.496 INFO:scrape: 2022-07-04T08:37:19.497 INFO:scrape:Failure: Test failure: test_multitenant_subvolumes (tasks.cephfs.test_volumes.TestSubvolumes), test_multitenant_subvolumes (tasks.cephfs.test_volumes.TestSubvolumes) 2022-07-04T08:37:19.497 INFO:scrape:1 jobs: ['6911918'] 2022-07-04T08:37:19.498 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/basic}}'] 2022-07-04T08:37:19.498 INFO:scrape: 2022-07-04T08:37:19.499 INFO:scrape:Failure: [Errno 113] No route to host 2022-07-04T08:37:19.500 INFO:scrape:2 jobs: ['6911981', '6911893'] 2022-07-04T08:37:19.500 INFO:scrape:suites intersection: ['1-ceph', '2-logrotate}', '2-workunit/suites/fsstress}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'objectstore-ec/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'races', 'session_timeout', 'thrashosds-health}'] 2022-07-04T08:37:19.501 INFO:scrape:suites union: ['1-ceph', '2-logrotate}', '2-workunit/suites/fsstress}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'objectstore-ec/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/3', 'ranks/5', 'session_timeout', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'thrashosds-health}'] 2022-07-04T08:37:19.501 INFO:scrape: 2022-07-04T08:37:19.502 INFO:scrape:Failure: Command failed on smithi117 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:61829bae536fb4126cfaa87ca353b9576e018e7d shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ce9be776-fb0c-11ec-842c-001a4aab830c -- ceph orch daemon add osd smithi117:vg_nvme/lv_4' 2022-07-04T08:37:19.502 INFO:scrape:2 jobs: ['6911973', '6911962'] 2022-07-04T08:37:19.503 INFO:scrape:suites intersection: ['1-cephadm', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'scrub/yes', 'session_timeout}', 'standby-replay', 'tasks/{0-check-counter', 'wsync/yes}'] 2022-07-04T08:37:19.504 INFO:scrape:suites union: ['1-cephadm', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/secure', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/1', 'ranks/3', 'scrub/yes', 'session_timeout}', 'standby-replay', 'subvolume/{with-namespace-isolated-and-quota}', 'subvolume/{with-quota}', 'tasks/{0-check-counter', 'workunit/suites/ffsb}}', 'workunit/suites/fsync-tester}}', 'wsync/yes}'] 2022-07-04T08:37:19.504 INFO:scrape: 2022-07-04T08:37:19.505 INFO:scrape:Crash: SSH connection to smithi112 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' 2022-07-04T08:37:19.505 INFO:scrape:ceph version 17.0.0-13322-g61829bae (61829bae536fb4126cfaa87ca353b9576e018e7d) quincy (dev) 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f32eafefce0] 2: gsignal() 3: abort() 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1b0) [0x7f32ed5e4fa2] 5: /usr/lib64/ceph/libceph-common.so.2(+0x2c2165) [0x7f32ed5e5165] 6: (Monitor::~Monitor()+0xb35) [0x556b2701a405] 7: (Monitor::~Monitor()+0xd) [0x556b2701a45d] 8: main() 9: __libc_start_main() 10: _start() 2022-07-04T08:37:19.506 INFO:scrape:4 jobs: ['6911970', '6911872', '6911920', '6911906'] 2022-07-04T08:37:19.506 INFO:scrape:suites intersection: ['1-ceph', '2-logrotate}', '2-workunit/suites/fsstress}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'races', 'ranks/5', 'session_timeout', 'thrashosds-health}'] 2022-07-04T08:37:19.507 INFO:scrape:suites union: ['1-ceph', '2-logrotate}', '2-workunit/suites/fsstress}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/5', 'session_timeout', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'thrashosds-health}'] 2022-07-04T08:37:19.507 INFO:scrape: 2022-07-04T08:37:19.508 INFO:scrape:Failure: Test failure: test_client_cache_size (tasks.cephfs.test_client_limits.TestClientLimits) 2022-07-04T08:37:19.508 INFO:scrape:1 jobs: ['6911969'] 2022-07-04T08:37:19.508 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/client-limits}'] 2022-07-04T08:37:19.509 INFO:scrape: 2022-07-04T08:37:19.509 INFO:scrape:Failure: error during scrub thrashing: reached maximum tries (30) after waiting for 900 seconds 2022-07-04T08:37:19.510 INFO:scrape:4 jobs: ['6911871', '6911895', '6911882', '6911921'] 2022-07-04T08:37:19.510 INFO:scrape:suites intersection: ['2-logrotate}', 'conf/{client', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{frag'] 2022-07-04T08:37:19.511 INFO:scrape:suites union: ['1-ceph', '1-cephadm', '2-logrotate}', '2-workunit/fs/trivial_sync}}', '2-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/thrash/workloads/{begin/{0-install', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/1', 'ranks/3', 'ranks/5', 'scrub/yes', 'session_timeout', 'session_timeout}', 'standby-replay', 'subvolume/{with-quota}', 'tasks/{0-check-counter', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'thrashosds-health}', 'workunit/fs/misc}}'] 2022-07-04T08:37:19.511 INFO:scrape: 2022-07-04T08:37:19.512 INFO:scrape:Failure: Test failure: test_multimds_hardlink (tasks.cephfs.test_snapshots.TestSnapshots), test_multimds_hardlink (tasks.cephfs.test_snapshots.TestSnapshots) 2022-07-04T08:37:19.512 INFO:scrape:1 jobs: ['6911924'] 2022-07-04T08:37:19.512 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/snapshots}'] 2022-07-04T08:37:19.513 INFO:scrape: 2022-07-04T08:37:19.513 INFO:scrape:Failure: {'smithi039.front.sepia.ceph.com': {'attempts': 12, 'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result", 'changed': True}} 2022-07-04T08:37:19.514 INFO:scrape:2 jobs: ['6911880', '6911980'] 2022-07-04T08:37:19.514 INFO:scrape:suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'osd}', 'overrides/{frag', 'prefetch_dirfrags/no', 'races', 'session_timeout', 'tasks/{1-thrash/mds', 'thrashosds-health}'] 2022-07-04T08:37:19.515 INFO:scrape:suites union: ['1-ceph', '2-logrotate}', '2-workunit/suites/ffsb}}', '2-workunit/suites/fsstress}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/1', 'ranks/5', 'rhel_8}', 'session_timeout', 'tasks/{1-thrash/mds', 'thrashosds-health}'] 2022-07-04T08:37:19.515 INFO:scrape: 2022-07-04T08:37:19.516 INFO:scrape:Failure: Command failed (workunit test fs/snaps/snaptest-multiple-capsnaps.sh) on smithi036 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=61829bae536fb4126cfaa87ca353b9576e018e7d TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh' 2022-07-04T08:37:19.516 INFO:scrape:4 jobs: ['6911974', '6911932', '6911960', '6911940'] 2022-07-04T08:37:19.517 INFO:scrape:suites intersection: ['1-ceph', '2-logrotate}', '2-workunit/fs/snaps}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'msgr-failures/none', 'osd}', 'overrides/{frag', 'races', 'ranks/1', 'session_timeout', 'thrashosds-health}'] 2022-07-04T08:37:19.517 INFO:scrape:suites union: ['1-ceph', '2-logrotate}', '2-workunit/fs/snaps}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{ubuntu_latest}', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'msgr-failures/none', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{frag', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/1', 'session_timeout', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'thrashosds-health}'] 2022-07-04T08:37:19.517 INFO:scrape: 2022-07-04T08:37:19.518 INFO:scrape:Failure: "2022-07-03T16:31:41.527109+0000 mon.a (mon.0) 341 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY)" in cluster log 2022-07-04T08:37:19.518 INFO:scrape:1 jobs: ['6911896'] 2022-07-04T08:37:19.519 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/multimds_misc}'] 2022-07-04T08:37:19.519 INFO:scrape: 2022-07-04T08:37:19.520 INFO:scrape:Failure: Command failed on smithi080 with status 123: "sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --" 2022-07-04T08:37:19.520 INFO:scrape:1 jobs: ['6911869'] 2022-07-04T08:37:19.520 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'centos_latest', 'clients/mirror', 'cluster/1-node', 'debug', 'fs/valgrind/{begin/{0-install', 'mirror/{cephfs-mirror/one-per-cluster', 'mount/fuse', 'overrides/whitelist_health', 'tasks/mirror}}'] 2022-07-04T08:37:19.521 INFO:scrape: 2022-07-04T08:37:19.521 INFO:scrape:Failure: Command failed (workunit test suites/pjd.sh) on smithi047 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=61829bae536fb4126cfaa87ca353b9576e018e7d TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh' 2022-07-04T08:37:19.522 INFO:scrape:2 jobs: ['6911913', '6911886'] 2022-07-04T08:37:19.522 INFO:scrape:suites intersection: ['1-cephadm', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{frag', 'scrub/yes', 'session_timeout}', 'standby-replay', 'tasks/{0-check-counter', 'workunit/suites/pjd}}'] 2022-07-04T08:37:19.523 INFO:scrape:suites union: ['1-cephadm', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{frag', 'ranks/1', 'ranks/3', 'rhel_8}', 'scrub/yes', 'session_timeout}', 'standby-replay', 'subvolume/{with-namespace-isolated-and-quota}', 'subvolume/{with-quota}', 'tasks/{0-check-counter', 'workunit/suites/pjd}}', 'wsync/no}'] 2022-07-04T08:37:19.523 INFO:scrape: 2022-07-04T08:37:19.524 INFO:scrape:Timeout 3h running clone.client.0/qa/workunits/fs/snaps/snaptest-git-ceph.sh 2022-07-04T08:37:19.524 INFO:scrape:1 jobs: ['6911972'] 2022-07-04T08:37:19.524 INFO:scrape:suites: ['1-ceph', '2-logrotate}', '2-workunit/fs/snaps}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8}', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'msgr-failures/none', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{frag', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/1', 'session_timeout', 'tasks/{1-thrash/osd', 'thrashosds-health}'] 2022-07-04T08:37:19.525 INFO:scrape: 2022-07-04T08:37:19.525 INFO:scrape:Failure: Test failure: test_scrub_checks (tasks.cephfs.test_scrub_checks.TestScrubChecks), test_scrub_checks (tasks.cephfs.test_scrub_checks.TestScrubChecks) 2022-07-04T08:37:19.526 INFO:scrape:1 jobs: ['6911914'] 2022-07-04T08:37:19.526 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/scrub}'] 2022-07-04T08:37:19.527 INFO:scrape: 2022-07-04T08:37:19.527 INFO:scrape:Failure: Test failure: test_dump_loads (tasks.cephfs.test_admin.TestAdminCommandDumpLoads) 2022-07-04T08:37:19.528 INFO:scrape:1 jobs: ['6911952'] 2022-07-04T08:37:19.528 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/admin}'] 2022-07-04T08:37:19.528 INFO:scrape: 2022-07-04T08:37:19.529 INFO:scrape:Crash: Cannot connect to remote host smithi117 2022-07-04T08:37:19.529 INFO:scrape:ceph version 17.0.0-13322-g61829bae (61829bae536fb4126cfaa87ca353b9576e018e7d) quincy (dev) 1: /lib64/libpthread.so.0(+0x12ce0) [0x7fbd85612ce0] 2: gsignal() 3: abort() 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x1b8) [0x55f59621c2d6] 5: (KernelDevice::_aio_thread()+0x1275) [0x55f596e7e585] 6: (KernelDevice::AioCompletionThread::entry()+0x11) [0x55f596e8bae1] 7: /lib64/libpthread.so.0(+0x81cf) [0x7fbd856081cf] 8: clone() 2022-07-04T08:37:19.530 INFO:scrape:1 jobs: ['6911889'] 2022-07-04T08:37:19.530 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'ranks/3', 'session_timeout}', 'tasks/fsstress', 'validater/valgrind}'] 2022-07-04T08:37:19.531 INFO:scrape: 2022-07-04T08:37:19.531 INFO:scrape:Failure: Cannot connect to remote host smithi204 2022-07-04T08:37:19.531 INFO:scrape:1 jobs: ['6911933'] 2022-07-04T08:37:19.532 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'fs/mixed-clients/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-bitmap', 'osd-asserts}', 'osd}', 'overrides/{ignorelist_health', 'tasks/kernel_cfuse_workunits_dbench_iozone}'] 2022-07-04T08:37:19.532 INFO:scrape: 2022-07-04T08:37:19.533 INFO:scrape:Failure: Test failure: test_r_with_fsname_and_no_path_in_cap (tasks.cephfs.test_multifs_auth.TestMDSCaps), test_r_with_fsname_and_no_path_in_cap (tasks.cephfs.test_multifs_auth.TestMDSCaps) 2022-07-04T08:37:19.533 INFO:scrape:1 jobs: ['6911982'] 2022-07-04T08:37:19.534 INFO:scrape:suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/multifs-auth}'] 2022-07-04T08:37:19.534 INFO:scrape: 2022-07-04T08:37:19.541 INFO:teuthology.results:Sending results to rishabhddave@gmail.com,ridave@redhat.com: Test Run: rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi ================================================================= info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/ logs: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/ failed: 56 dead: 34 running: 1 waiting: 0 queued: 0 passed: 30 Fail ================================================================= [6911869] fs/valgrind/{begin/{0-install 1-ceph 2-logrotate} centos_latest debug mirror/{cephfs-mirror/one-per-cluster clients/mirror cluster/1-node mount/fuse overrides/whitelist_health tasks/mirror}} ----------------------------------------------------------------- time: 02:01:06 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911869/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911869/ Command failed on smithi080 with status 123: "sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --" [6911871] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay subvolume/{with-quota} tasks/{0-check-counter workunit/fs/misc}} ----------------------------------------------------------------- time: 01:20:43 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911871/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911871/ error during scrub thrashing: reached maximum tries (30) after waiting for 900 seconds [6911872] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:42:56 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911872/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911872/ sentry: https://sentry.ceph.com/organizations/ceph/?query=b166e1ece8b345fe96b33aa8e67013a8 SSH connection to smithi145 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911874] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:26:13 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911874/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911874/ sentry: https://sentry.ceph.com/organizations/ceph/?query=8612074071eb490fbfdac156954a7233 SSH connection to smithi163 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911879] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:33:11 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911879/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911879/ sentry: https://sentry.ceph.com/organizations/ceph/?query=25953f67c16e4d068fe3aa76d03d5f03 SSH connection to smithi070 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911882] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:25:19 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911882/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911882/ sentry: https://sentry.ceph.com/organizations/ceph/?query=5743a85bfe904f9184df64f56c0be31a reached maximum tries (90) after waiting for 540 seconds [6911883] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:33:38 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911883/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911883/ sentry: https://sentry.ceph.com/organizations/ceph/?query=d17e1650638040a597bf8f0251b89d47 SSH connection to smithi123 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911886] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay subvolume/{with-quota} tasks/{0-check-counter workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:24:05 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911886/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911886/ sentry: https://sentry.ceph.com/organizations/ceph/?query=d0022792a2be46bbbee837a350d13168 Command failed (workunit test suites/pjd.sh) on smithi149 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=61829bae536fb4126cfaa87ca353b9576e018e7d TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh' [6911888] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:27:04 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911888/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911888/ sentry: https://sentry.ceph.com/organizations/ceph/?query=d017c36eecde4638a802edc9a0d40c36 SSH connection to smithi039 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911889] fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/3 tasks/fsstress validater/valgrind} ----------------------------------------------------------------- time: 01:23:26 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911889/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911889/ sentry: https://sentry.ceph.com/organizations/ceph/?query=80b453cfd31e4004b6df44f91d3305b6 Cannot connect to remote host smithi117 [6911893] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 06:34:39 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911893/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911893/ sentry: https://sentry.ceph.com/organizations/ceph/?query=3dd6be9c91a24661b48bb797c5619f07 [Errno 113] No route to host [6911895] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/trivial_sync}} ----------------------------------------------------------------- time: 00:20:52 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911895/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911895/ sentry: https://sentry.ceph.com/organizations/ceph/?query=24f096b18f0c48deba43df1e9b448786 reached maximum tries (90) after waiting for 540 seconds [6911896] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/multimds_misc} ----------------------------------------------------------------- time: 00:14:23 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911896/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911896/ "2022-07-03T16:31:41.527109+0000 mon.a (mon.0) 341 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY)" in cluster log [6911899] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/iozone}} ----------------------------------------------------------------- time: 00:40:04 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911899/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911899/ sentry: https://sentry.ceph.com/organizations/ceph/?query=33ac10796cd74aeca1057fd5425c5d2d SSH connection to smithi035 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.1 /home/ubuntu/cephtest/clone.client.1' [6911900] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 03:08:50 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911900/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911900/ sentry: https://sentry.ceph.com/organizations/ceph/?query=a1accfa4a03b4698b9c82226a3a1d823 SSH connection to smithi133 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911906] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:25:54 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911906/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911906/ sentry: https://sentry.ceph.com/organizations/ceph/?query=d12a127d97f04748b7a9928e6756567f SSH connection to smithi125 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911909] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 02:24:46 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911909/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911909/ sentry: https://sentry.ceph.com/organizations/ceph/?query=bf6801ddf1674a1a93b462a7e3dd1dbe SSH connection to smithi123 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911911] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:27:20 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911911/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911911/ sentry: https://sentry.ceph.com/organizations/ceph/?query=6f939fd1a1354cc4aa8316d63ad1d70d SSH connection to smithi145 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911913] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/legacy wsync/no} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/yes standby-replay subvolume/{with-namespace-isolated-and-quota} tasks/{0-check-counter workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:23:39 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911913/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911913/ sentry: https://sentry.ceph.com/organizations/ceph/?query=3739e27e6e714928bfc71ca62a543cc8 Command failed (workunit test suites/pjd.sh) on smithi047 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=61829bae536fb4126cfaa87ca353b9576e018e7d TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh' [6911914] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/scrub} ----------------------------------------------------------------- time: 00:26:26 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911914/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911914/ sentry: https://sentry.ceph.com/organizations/ceph/?query=09afc5b8678d4ba99324d48a32e3e2f1 Test failure: test_scrub_checks (tasks.cephfs.test_scrub_checks.TestScrubChecks), test_scrub_checks (tasks.cephfs.test_scrub_checks.TestScrubChecks) [6911918] fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/basic}} ----------------------------------------------------------------- time: 02:30:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911918/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911918/ sentry: https://sentry.ceph.com/organizations/ceph/?query=63592873a8404b348220254e01244db1 Test failure: test_multitenant_subvolumes (tasks.cephfs.test_volumes.TestSubvolumes), test_multitenant_subvolumes (tasks.cephfs.test_volumes.TestSubvolumes) [6911920] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:27:11 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911920/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911920/ sentry: https://sentry.ceph.com/organizations/ceph/?query=9289625e03b84a5db749d9413dbad3e2 SSH connection to smithi125 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911921] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:19:11 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911921/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911921/ sentry: https://sentry.ceph.com/organizations/ceph/?query=d16aa1b9171d448d9b2afa097f025eef reached maximum tries (90) after waiting for 540 seconds [6911922] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay subvolume/{with-namespace-isolated} tasks/{0-check-counter workunit/suites/blogbench}} ----------------------------------------------------------------- time: 00:42:47 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911922/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911922/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ff2e1253d96040b7a043fb923c16ee0c SSH connection to smithi136 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911923] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:26:49 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911923/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911923/ sentry: https://sentry.ceph.com/organizations/ceph/?query=8c1577905e2848b0b790c12ab69346f9 SSH connection to smithi042 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911924] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/snapshots} ----------------------------------------------------------------- time: 00:34:18 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911924/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911924/ sentry: https://sentry.ceph.com/organizations/ceph/?query=873b336857a345628dcd203822c85139 Test failure: test_multimds_hardlink (tasks.cephfs.test_snapshots.TestSnapshots), test_multimds_hardlink (tasks.cephfs.test_snapshots.TestSnapshots) [6911932] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 00:53:09 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911932/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911932/ sentry: https://sentry.ceph.com/organizations/ceph/?query=97fabded531344358b6c1e16f6ede353 Command failed (workunit test fs/snaps/snaptest-multiple-capsnaps.sh) on smithi112 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=61829bae536fb4126cfaa87ca353b9576e018e7d TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snaptest- multiple-capsnaps.sh' [6911933] fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/k-testing ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_dbench_iozone} ----------------------------------------------------------------- time: 00:28:56 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911933/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911933/ sentry: https://sentry.ceph.com/organizations/ceph/?query=710ad640a2ec41308b3cc14e34c57b8d Cannot connect to remote host smithi204 [6911935] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:27:49 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911935/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911935/ sentry: https://sentry.ceph.com/organizations/ceph/?query=05bb0a020dde4f4e89ff475f5e69aca5 SSH connection to smithi006 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911938] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 02:01:30 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911938/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911938/ sentry: https://sentry.ceph.com/organizations/ceph/?query=b58c7f7770a6459f9e47a3bf68ab44a5 SSH connection to smithi050 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911940] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 01:01:52 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911940/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911940/ sentry: https://sentry.ceph.com/organizations/ceph/?query=c2d4baaf422a47d28553a2ece2e5290d Command failed (workunit test fs/snaps/snaptest-multiple-capsnaps.sh) on smithi137 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=61829bae536fb4126cfaa87ca353b9576e018e7d TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snaptest- multiple-capsnaps.sh' [6911946] fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/1 tasks/fsstress validater/valgrind} ----------------------------------------------------------------- time: 01:11:46 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911946/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911946/ sentry: https://sentry.ceph.com/organizations/ceph/?query=63b7534785cd44bba40e6e0998884268 SSH connection to smithi117 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911952] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/admin} ----------------------------------------------------------------- time: 00:15:50 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911952/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911952/ sentry: https://sentry.ceph.com/organizations/ceph/?query=bfa76b2676aa44d0a2590471f05eafa1 Test failure: test_dump_loads (tasks.cephfs.test_admin.TestAdminCommandDumpLoads) [6911954] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/yes standby-replay subvolume/{with-no-extra-options} tasks/{0-check-counter workunit/suites/ffsb}} ----------------------------------------------------------------- time: 00:59:40 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911954/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911954/ sentry: https://sentry.ceph.com/organizations/ceph/?query=a88e0e6792b8481687c6187b6ccf00b7 Command crashed: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911956] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:26:34 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911956/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911956/ sentry: https://sentry.ceph.com/organizations/ceph/?query=3f072934ef4b49a784d9ee660aa50018 SSH connection to smithi189 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911958] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:31:56 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911958/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911958/ sentry: https://sentry.ceph.com/organizations/ceph/?query=5343d7c85dc14829a7b07058caa38a2e SSH connection to smithi112 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911960] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 00:49:15 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911960/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911960/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ab7e16c239e342dc913f4fceb1325bc7 Command failed (workunit test fs/snaps/snaptest-multiple-capsnaps.sh) on smithi079 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=61829bae536fb4126cfaa87ca353b9576e018e7d TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snaptest- multiple-capsnaps.sh' [6911962] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/yes standby-replay subvolume/{with-namespace-isolated-and-quota} tasks/{0-check-counter workunit/suites/fsync-tester}} ----------------------------------------------------------------- time: 00:47:03 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911962/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911962/ sentry: https://sentry.ceph.com/organizations/ceph/?query=f4bb6ff1faf74ba1a966b8bfecf018bd Command failed on smithi137 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph- ci/ceph:61829bae536fb4126cfaa87ca353b9576e018e7d pull' [6911963] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:32:44 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911963/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911963/ sentry: https://sentry.ceph.com/organizations/ceph/?query=5dba08cc9d6748d2bc724a53c4ab1e31 SSH connection to smithi189 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911964] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 04:29:54 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911964/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911964/ sentry: https://sentry.ceph.com/organizations/ceph/?query=57b71ad101c14049aa36688c182211f2 Command crashed: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911969] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client-limits} ----------------------------------------------------------------- time: 00:13:18 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911969/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911969/ sentry: https://sentry.ceph.com/organizations/ceph/?query=7e6e826b00644aefa47fba004e95d807 Test failure: test_client_cache_size (tasks.cephfs.test_client_limits.TestClientLimits) [6911970] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:27:12 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911970/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911970/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ff4e0f6c84cf4047a5d8aa991353d64f SSH connection to smithi112 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911971] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:25:33 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911971/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911971/ sentry: https://sentry.ceph.com/organizations/ceph/?query=2ee913a3948c44aea59a329dc44efa34 SSH connection to smithi133 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911972] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 04:04:09 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911972/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911972/ sentry: https://sentry.ceph.com/organizations/ceph/?query=44b2dec0cc764a9886b3a7df7f1c299b Command failed (workunit test fs/snaps/snaptest-git-ceph.sh) on smithi083 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=61829bae536fb4126cfaa87ca353b9576e018e7d TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snaptest-git- ceph.sh' [6911973] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay subvolume/{with-quota} tasks/{0-check-counter workunit/suites/ffsb}} ----------------------------------------------------------------- time: 00:17:45 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911973/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911973/ sentry: https://sentry.ceph.com/organizations/ceph/?query=ad411ca6ca8f43788a513f6e3a6f19cd Command failed on smithi117 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph- ci/ceph:61829bae536fb4126cfaa87ca353b9576e018e7d shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ce9be776-fb0c-11ec-842c-001a4aab830c -- ceph orch daemon add osd smithi117:vg_nvme/lv_4' [6911974] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/snaps}} ----------------------------------------------------------------- time: 00:33:07 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911974/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911974/ sentry: https://sentry.ceph.com/organizations/ceph/?query=285559701e724451bb17b29fa9ff2b4f Command failed (workunit test fs/snaps/snaptest-multiple-capsnaps.sh) on smithi036 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=61829bae536fb4126cfaa87ca353b9576e018e7d TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snaptest- multiple-capsnaps.sh' [6911977] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:25:54 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911977/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911977/ sentry: https://sentry.ceph.com/organizations/ceph/?query=c2300a991fe042458ca872f019a82d00 SSH connection to smithi047 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911978] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 01:15:30 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911978/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911978/ sentry: https://sentry.ceph.com/organizations/ceph/?query=68bb8561f5ca441392d32cd0376aa34f SSH connection to smithi189 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911979] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/yes standby-replay subvolume/{with-namespace-isolated} tasks/{0-check-counter workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:41:24 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911979/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911979/ sentry: https://sentry.ceph.com/organizations/ceph/?query=294bb35c878947ffa4973f8b0a3524fa SSH connection to smithi102 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911981] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:27:02 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911981/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911981/ sentry: https://sentry.ceph.com/organizations/ceph/?query=418b563844b9434fb4506853313e3ae6 [Errno 113] No route to host [6911982] fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/multifs-auth} ----------------------------------------------------------------- time: 00:34:58 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911982/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911982/ sentry: https://sentry.ceph.com/organizations/ceph/?query=37953d70b77a44869ad5ef10c3e1e0c0 Test failure: test_r_with_fsname_and_no_path_in_cap (tasks.cephfs.test_multifs_auth.TestMDSCaps), test_r_with_fsname_and_no_path_in_cap (tasks.cephfs.test_multifs_auth.TestMDSCaps) [6911983] fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} ----------------------------------------------------------------- time: 00:26:07 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911983/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911983/ sentry: https://sentry.ceph.com/organizations/ceph/?query=7be3200f4f3f4956a3ee1d694fd93851 SSH connection to smithi052 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911984] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:26:42 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911984/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911984/ sentry: https://sentry.ceph.com/organizations/ceph/?query=de1ea52f09704f59ae2ed12a564b510b SSH connection to smithi112 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911986] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:26:20 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911986/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911986/ sentry: https://sentry.ceph.com/organizations/ceph/?query=14f914a4ba4e4d4d96a09b8bc5e85465 SSH connection to smithi050 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911987] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 05:18:28 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911987/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911987/ sentry: https://sentry.ceph.com/organizations/ceph/?query=5d60d45506604f839defe6204a132c8f SSH connection to smithi111 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' [6911988] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/yes standby-replay subvolume/{with-namespace-isolated-and-quota} tasks/{0-check-counter workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:41:45 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911988/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911988/ sentry: https://sentry.ceph.com/organizations/ceph/?query=c4f1f9d55e18447cb78e2261c9fd84fb SSH connection to smithi036 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' Dead ================================================================= [6911868] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911868/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911868/ hit max job timeout [6911870] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/no standby-replay subvolume/{with-no-extra-options} tasks/{0-check-counter workunit/direct_io}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911870/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911870/ hit max job timeout [6911873] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/no standby-replay subvolume/{with-no-extra-options} tasks/{0-check-counter workunit/kernel_untar_build}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911873/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911873/ hit max job timeout [6911877] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/no standby-replay subvolume/{with-namespace-isolated} tasks/{0-check-counter workunit/suites/dbench}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911877/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911877/ hit max job timeout [6911878] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911878/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911878/ hit max job timeout [6911880] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:08:44 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911880/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911880/ sentry: https://sentry.ceph.com/organizations/ceph/?query=36de91abe9224f55a967ff5d5ff32e99 {'smithi039.front.sepia.ceph.com': {'attempts': 12, 'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result", 'changed': True}} [6911881] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/no standby-replay subvolume/{with-quota} tasks/{0-check-counter workunit/suites/fsx}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911881/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911881/ hit max job timeout [6911885] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911885/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911885/ hit max job timeout [6911890] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/no standby-replay subvolume/{with-namespace-isolated-and-quota} tasks/{0-check-counter workunit/direct_io}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911890/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911890/ hit max job timeout [6911891] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911891/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911891/ hit max job timeout [6911892] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/no standby-replay subvolume/{no-subvolume} tasks/{0-check-counter workunit/kernel_untar_build}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911892/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911892/ hit max job timeout [6911897] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/no standby-replay subvolume/{with-namespace-isolated} tasks/{0-check-counter workunit/suites/dbench}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911897/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911897/ hit max job timeout [6911898] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911898/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911898/ hit max job timeout [6911903] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911903/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911903/ hit max job timeout [6911904] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/no standby-replay subvolume/{with-namespace-isolated} tasks/{0-check-counter workunit/suites/fsx}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911904/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911904/ hit max job timeout [6911905] fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/k-testing ms-die-on-skipped} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_dbench_iozone} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911905/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911905/ hit max job timeout [6911915] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911915/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911915/ hit max job timeout [6911917] fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_snaptests}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911917/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911917/ hit max job timeout [6911925] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/yes standby-replay subvolume/{with-namespace-isolated} tasks/{0-check-counter workunit/suites/ffsb}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911925/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911925/ hit max job timeout [6911927] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911927/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911927/ hit max job timeout [6911929] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/yes standby-replay subvolume/{with-quota} tasks/{0-check-counter workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911929/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911929/ hit max job timeout [6911930] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911930/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911930/ hit max job timeout [6911931] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911931/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911931/ hit max job timeout [6911941] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/yes standby-replay subvolume/{no-subvolume} tasks/{0-check-counter workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911941/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911941/ hit max job timeout [6911943] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911943/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911943/ hit max job timeout [6911948] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911948/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911948/ hit max job timeout [6911950] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911950/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911950/ hit max job timeout [6911951] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/yes standby-replay subvolume/{with-quota} tasks/{0-check-counter workunit/suites/blogbench}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911951/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911951/ hit max job timeout [6911953] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/pjd}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911953/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911953/ hit max job timeout [6911957] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay subvolume/{with-namespace-isolated} tasks/{0-check-counter workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911957/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911957/ hit max job timeout [6911959] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/asok_dump_tree} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911959/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911959/ hit max job timeout [6911967] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911967/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911967/ hit max job timeout [6911976] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/yes standby-replay subvolume/{with-no-extra-options} tasks/{0-check-counter workunit/suites/fsstress}} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911976/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911976/ hit max job timeout [6911980] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} ----------------------------------------------------------------- time: 00:06:48 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911980/ log: http://qa-proxy.ceph.com/teuthology/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911980/ sentry: https://sentry.ceph.com/organizations/ceph/?query=3b415b51a7744e20abbb7927b6044122 {'smithi050.front.sepia.ceph.com': {'attempts': 12, 'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result", 'changed': True}} Running ================================================================= [6911916] fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911916/ Pass ================================================================= [6911875] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:28:09 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911875/ [6911876] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} time: 00:41:10 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911876/ [6911884] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mds-full} time: 00:21:16 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911884/ [6911887] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:28:02 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911887/ [6911894] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/yes standby-replay subvolume/{with-namespace-isolated-and-quota} tasks/{0-check-counter workunit/suites/blogbench}} time: 00:40:47 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911894/ [6911901] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:28:06 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911901/ [6911902] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/trivial_sync}} time: 00:13:38 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911902/ [6911907] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/trivial_sync}} time: 00:10:01 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911907/ [6911908] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/no standby-replay subvolume/{with-namespace-isolated-and-quota} tasks/{0-check-counter workunit/fs/test_o_trunc}} time: 00:38:24 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911908/ [6911910] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay subvolume/{no-subvolume} tasks/{0-check-counter workunit/suites/iogen}} time: 00:38:38 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911910/ [6911912] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:29:04 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911912/ [6911919] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/snaps}} time: 01:32:49 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911919/ [6911926] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:28:07 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911926/ [6911928] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/no standby-replay subvolume/{with-namespace-isolated} tasks/{0-check-counter workunit/fs/norstats}} time: 00:26:05 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911928/ [6911934] fs/traceless/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_blogbench traceless/50pc} time: 00:22:33 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911934/ [6911936] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/legacy wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/no standby-replay subvolume/{with-quota} tasks/{0-check-counter workunit/suites/iozone}} time: 00:24:23 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911936/ [6911937] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/workunit/dir-max-entries} time: 00:10:28 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911937/ [6911939] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:28:33 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911939/ [6911942] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} time: 00:40:04 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911942/ [6911944] fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} time: 00:14:18 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911944/ [6911945] fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} time: 01:20:35 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911945/ [6911947] fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay subvolume/{with-namespace-isolated} tasks/{0-check-counter workunit/fs/misc}} time: 01:02:18 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911947/ [6911949] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/snaps}} time: 01:20:29 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911949/ [6911955] fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/alternate-pool} time: 00:12:34 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911955/ [6911961] fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} time: 00:28:28 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911961/ [6911965] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no prefetch_entire_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/snaps}} time: 01:17:25 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911965/ [6911966] fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_untarbuild_blogbench} time: 00:47:35 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911966/ [6911968] fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/failover} time: 00:52:55 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911968/ [6911975] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/ffsb}} time: 00:22:00 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911975/ [6911985] fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes prefetch_entire_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/fs/snaps}} time: 01:16:46 info: http://pulpito.ceph.com/rishabh-2022-07-03_15:10:28-fs-wip-rishabh-testing-20220702-1631-distro-default-smithi/6911985/