Found 178 jobs Missing teuthology log /home/teuthworker/archive/yuriw-2023-11-18_13:55:06-rbd-main-distro-default-smithi/7461986/teuthology.log Found 10 distinct failure reasons Failure: Command failed (workunit test rbd/rbd_mirror.sh) on smithi170 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b0f45a51a24e8eca7cf153dba57cbb536ce5b3fe TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS=\'\' RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=journal RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh' 1 jobs: ['7462220'] suites: ['clients/mirror', 'cluster/{2-node', 'conf/{disable-pool-app}', 'msgr-failures/few', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'policy/none', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{centos_latest}', 'workloads/rbd-mirror-journal-workunit}'] Dead: b'2023-11-19T15:03:53.289 DEBUG:teuthology.exit:Finished running handlers' 2 jobs: ['7461928', '7462088'] suites intersection: ['openstack}'] suites union: ['2-clusters/{fixed-3', '3-objectstore/bluestore-stupid', '4-supported-random-distro$/{ubuntu_latest}', '5-data-pool/none', '6-prepare/qcow2-file', '7-io-workloads/qemu_xfstests', '8-migrate-workloads/execute', '9-cleanup/cleanup', 'clusters/{fixed-3', 'conf/{disable-pool-app}', 'conf/{disable-pool-app}}', 'data-pool/replicated', 'features/journaling', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rbd/migration/{1-base/install', 'rbd/qemu/{cache/writethrough', 'supported-random-distro$/{ubuntu_latest}', 'workloads/qemu_xfstests}'] Timeout 120 running ceph --cluster cluster2 pg dump --format=json 1 jobs: ['7461914'] suites: ['clients/mirror', 'cluster/{2-node', 'conf/{disable-pool-app}', 'msgr-failures/few', 'objectstore/bluestore-hybrid', 'openstack}', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd-mirror-snapshot-stress-workunit-minimum}'] Crash: Command failed on smithi175 with status 139: 'sudo TESTDIR=/home/ubuntu/cephtest bash -c \'echo \'"\'"\'{"type":"qcow","stream":{"type":"http","url":"http://download.ceph.com/qa/ubuntu-12.04.qcow2"}}\'"\'"\' | rbd migration prepare --import-only --source-spec-path - client.0.0\'' ceph version 18.0.0-7418-gb0f45a51 (b0f45a51a24e8eca7cf153dba57cbb536ce5b3fe) reef (dev) 1: /lib64/libc.so.6(+0x54db0) [0x7fc032c54db0] 2: /lib64/librbd.so.1(+0x2f5e41) [0x7fc0342f5e41] 3: /lib64/librados.so.2(+0x12f5ae) [0x7fc033f785ae] 4: /lib64/librados.so.2(+0xcc0df) [0x7fc033f150df] 5: /lib64/libstdc++.so.6(+0xdb924) [0x7fc0330db924] 6: /lib64/libc.so.6(+0x9f802) [0x7fc032c9f802] 7: /lib64/libc.so.6(+0x3f450) [0x7fc032c3f450] 1 jobs: ['7462124'] suites: ['2-clusters/{fixed-3', '3-objectstore/bluestore-bitmap', '4-supported-random-distro$/{centos_latest}', '5-data-pool/replicated', '6-prepare/qcow2-http', '7-io-workloads/qemu_xfstests', '8-migrate-workloads/execute', '9-cleanup/cleanup', 'conf/{disable-pool-app}}', 'openstack}', 'rbd/migration/{1-base/install'] Crash: Command failed on smithi154 with status 139: 'sudo TESTDIR=/home/ubuntu/cephtest bash -c \'echo \'"\'"\'{"type":"qcow","stream":{"type":"http","url":"http://download.ceph.com/qa/ubuntu-12.04.qcow2"}}\'"\'"\' | rbd migration prepare --import-only --source-spec-path - client.0.0\'' ceph version 18.0.0-7418-gb0f45a51 (b0f45a51a24e8eca7cf153dba57cbb536ce5b3fe) reef (dev) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7faf4c9fd520] 2: /lib/librbd.so.1(+0x2e1a81) [0x7faf4dd7da81] 3: /lib/librados.so.2(+0x111e2e) [0x7faf4da22e2e] 4: /lib/librados.so.2(+0xc268f) [0x7faf4d9d368f] 5: /lib/x86_64-linux-gnu/libstdc++.so.6(+0xdc2b3) [0x7faf4cdc82b3] 6: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7faf4ca4fb43] 7: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7faf4cae1a00] 1 jobs: ['7462006'] suites: ['2-clusters/{fixed-3', '3-objectstore/bluestore-hybrid', '4-supported-random-distro$/{ubuntu_latest}', '5-data-pool/replicated', '6-prepare/qcow2-http', '7-io-workloads/qemu_xfstests', '8-migrate-workloads/execute', '9-cleanup/cleanup', 'conf/{disable-pool-app}}', 'openstack}', 'rbd/migration/{1-base/install'] Failure: Command failed on smithi138 with status 1: 'test -f /home/ubuntu/cephtest/archive/qemu/client.0/success' 3 jobs: ['7462138', '7461954', '7462207'] suites intersection: ['2-cluster/{fix-2', '4-cache-path', '7-workloads/qemu_xfstests', 'conf/{disable-pool-app}}', 'openstack}', 'rbd/pwl-cache/tmpfs/{1-base/install'] suites union: ['2-cluster/{fix-2', '3-supported-random-distro$/{centos_latest}', '3-supported-random-distro$/{ubuntu_latest}', '4-cache-path', '5-cache-mode/rwl', '5-cache-mode/ssd', '6-cache-size/1G', '6-cache-size/5G', '7-workloads/qemu_xfstests', 'conf/{disable-pool-app}}', 'openstack}', 'rbd/pwl-cache/tmpfs/{1-base/install'] Dead: None 1 jobs: ['7461986'] Failure: ['E: Could not get lock /var/cache/apt/archives/lock. It is held by process 27887 (apt-get)', 'E: Unable to lock directory /var/cache/apt/archives/'] 1 jobs: ['7462090'] suites: ['cluster/{fixed-3', 'conf/{disable-pool-app}', 'msgr-failures/few', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rbd/nbd/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/default', 'thrashosds-health', 'workloads/rbd_fsx_nbd}'] Failure: No module named 'tasks' 1 jobs: ['7462158'] suites: ['clusters/{fixed-3', 'conf/{disable-pool-app}', 'data-pool/ec', 'features/defaults', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'openstack}', 'rbd/encryption/{cache/writeback', 'supported-random-distro$/{centos_latest}', 'workloads/qemu_xfstests_luks2_luks2}'] Crash: Command crashed: "sudo TESTDIR=/home/ubuntu/cephtest bash -c 'rbd migration execute client.0.0'" ceph version 18.0.0-7418-gb0f45a51 (b0f45a51a24e8eca7cf153dba57cbb536ce5b3fe) reef (dev) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f7acf046520] 2: /lib/librbd.so.1(+0x2e1a81) [0x7f7ad03c6a81] 3: /lib/librados.so.2(+0x111e2e) [0x7f7ad006be2e] 4: /lib/librados.so.2(+0xc268f) [0x7f7ad001c68f] 5: /lib/x86_64-linux-gnu/libstdc++.so.6(+0xdc2b3) [0x7f7acf4112b3] 6: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7f7acf098b43] 7: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7f7acf12aa00] 1 jobs: ['7462216'] suites: ['2-clusters/{fixed-3', '3-objectstore/bluestore-comp-zlib', '4-supported-random-distro$/{ubuntu_latest}', '5-data-pool/replicated', '6-prepare/qcow2-http', '7-io-workloads/qemu_xfstests', '8-migrate-workloads/execute', '9-cleanup/cleanup', 'conf/{disable-pool-app}}', 'openstack}', 'rbd/migration/{1-base/install']