2023-06-15T01:52:06.831 INFO:root:teuthology version: 0.0.1.dev124+g961f4fb 2023-06-15T01:52:06.832 INFO:teuthology.results:seed: '9257' 2023-06-15T02:22:30.695 INFO:root:teuthology version: 0.0.1.dev124+g961f4fb 2023-06-15T02:22:30.699 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2023-06-15T14:23:28.460 WARNING:teuthology.results:test(s) did not finish before timeout of 43200 seconds 2023-06-15T14:23:29.668 INFO:scrape:Found 20 jobs 2023-06-15T14:23:31.175 INFO:scrape:Found 6 distinct failure reasons 2023-06-15T14:23:31.176 INFO:scrape:Dead: b'2023-06-15T14:13:35.730 DEBUG:teuthology.exit:Finished running handlers' 2023-06-15T14:23:31.176 INFO:scrape:6 jobs: ['7304978', '7304981', '7304979', '7304982', '7304974', '7304971'] 2023-06-15T14:23:31.177 INFO:scrape:suites intersection: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health'] 2023-06-15T14:23:31.177 INFO:scrape:suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/admin_socket_objecter_requests}', 'workloads/radosbench-high-concurrency}', 'workloads/small-objects-balanced}', 'workloads/small-objects-localized}', 'workloads/small-objects}', 'workloads/snaps-few-objects-balanced}'] 2023-06-15T14:23:31.178 INFO:scrape: 2023-06-15T14:23:31.178 INFO:scrape:Timeout 3h running clone.client.0/qa/workunits/rbd/test_librbd_python.sh --eval-attr \ 2023-06-15T14:23:31.178 INFO:scrape:1 jobs: ['7304984'] 2023-06-15T14:23:31.179 INFO:scrape:suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_python_api_tests}'] 2023-06-15T14:23:31.179 INFO:scrape: 2023-06-15T14:23:31.179 INFO:scrape:Timeout 3h running clone.client.0/qa/workunits/rbd/crimson/test_crimson_librbd.sh 2023-06-15T14:23:31.180 INFO:scrape:1 jobs: ['7304970'] 2023-06-15T14:23:31.180 INFO:scrape:suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_api_tests}'] 2023-06-15T14:23:31.180 INFO:scrape: 2023-06-15T14:23:31.181 INFO:scrape:Dead: b'2023-06-15T14:23:28.505 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1052,"fsid":"791bfd97-f02b-477e-8744-afa52378319e","created":"2023-06-15T02:39:54.690444+0000","modified":"2023-06-15T02:57:54.941364+0000","last_up_change":"2023-06-15T02:40:32.768978+0000","last_in_change":"2023-06-15T02:57:49.940095+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-15T02:40:04.990631+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-15T02:40:05.238964+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"82b56988-f4a1-46db-a5a3-f7b7049cbfb7","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":32,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6801","nonce":2198226803}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":2198226803}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":2198226803}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6805","nonce":2198226803}]},"public_addr":"172.21.15.136:6801/2198226803","cluster_addr":"172.21.15.136:6800/2198226803","heartbeat_back_addr":"172.21.15.136:6804/2198226803","heartbeat_front_addr":"172.21.15.136:6805/2198226803","state":["autoout","exists"]},{"osd":1,"uuid":"eaf1b406-792c-4084-9d3e-4f32cc3c6be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":1047,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6803","nonce":1767014796}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":1767014796}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6807","nonce":1767014796}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":1767014796}]},"public_addr":"172.21.15.136:6803/1767014796","cluster_addr":"172.21.15.136:6802/1767014796","heartbeat_back_addr":"172.21.15.136:6807/1767014796","heartbeat_front_addr":"172.21.15.136:6806/1767014796","state":["exists","up"]},{"osd":2,"uuid":"01b8ef59-bf37-4aa7-8ca8-8459d0369f57","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":1047,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":1538820502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":1538820502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":1538820502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":1538820502}]},"public_addr":"172.21.15.153:6802/1538820502","cluster_addr":"172.21.15.153:6803/1538820502","heartbeat_back_addr":"172.21.15.153:6807/1538820502","heartbeat_front_addr":"172.21.15.153:6806/1538820502","state":["exists","up"]},{"osd":3,"uuid":"97f92f79-7596-4bd7-9f5f-8c4ce78f3806","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":1047,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":3737539985}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":3737539985}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":3737539985}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":3737539985}]},"public_addr":"172.21.15.153:6804/3737539985","cluster_addr":"172.21.15.153:6805/3737539985","heartbeat_back_addr":"172.21.15.153:6808/3737539985","heartbeat_front_addr":"172.21.15.153:6809/3737539985","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-15T02:40:32.768978+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":3},{"begin":623,"length":3},{"begin":628,"length":5},{"begin":634,"length":4},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":656,"length":5},{"begin":665,"length":2},{"begin":670,"length":6},{"begin":678,"length":1},{"begin":680,"length":2},{"begin":685,"length":1},{"begin":693,"length":1},{"begin":696,"length":1},{"begin":699,"length":1},{"begin":702,"length":1},{"begin":704,"length":2},{"begin":707,"length":1},{"begin":709,"length":4},{"begin":714,"length":1},{"begin":716,"length":1},{"begin":718,"length":1},{"begin":720,"length":4},{"begin":725,"length":1},{"begin":728,"length":1},{"begin":730,"length":1},{"begin":732,"length":3},{"begin":737,"length":2},{"begin":741,"length":8},{"begin":750,"length":4},{"begin":755,"length":2},{"begin":759,"length":1},{"begin":761,"length":1},{"begin":763,"length":2},{"begin":767,"length":1},{"begin":771,"length":1},{"begin":781,"length":2},{"begin":787,"length":1},{"begin":790,"length":1},{"begin":792,"length":3},{"begin":796,"length":1},{"begin":799,"length":1},{"begin":801,"length":1},{"begin":806,"length":1},{"begin":808,"length":1},{"begin":812,"length":2},{"begin":815,"length":2},{"begin":818,"length":1},{"begin":820,"length":5},{"begin":827,"length":1},{"begin":831,"length":2},{"begin":834,"length":1},{"begin":836,"length":1},{"begin":839,"length":4},{"begin":844,"length":4},{"begin":850,"length":1},{"begin":852,"length":2},{"begin":855,"length":1},{"begin":862,"length":1},{"begin":866,"length":6},{"begin":873,"length":1},{"begin":876,"length":1},{"begin":880,"length":5},{"begin":886,"length":2},{"begin":889,"length":2},{"begin":893,"length":4},{"begin":903,"length":1},{"begin":905,"length":2},{"begin":911,"length":1},{"begin":913,"length":1},{"begin":920,"length":1},{"begin":922,"length":4},{"begin":927,"length":1},{"begin":930,"length":2},{"begin":934,"length":1},{"begin":937,"length":2},{"begin":940,"length":2},{"begin":953,"length":1},{"begin":956,"length":2},{"begin":960,"length":1},{"begin":962,"length":4},{"begin":967,"length":1},{"begin":972,"length":2},{"begin":977,"length":4},{"begin":988,"length":1},{"begin":992,"length":1},{"begin":994,"length":1},{"begin":999,"length":1},{"begin":1005,"length":2},{"begin":1012,"length":2},{"begin":1015,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}' 2023-06-15T14:23:31.181 INFO:scrape:1 jobs: ['7304986'] 2023-06-15T14:23:31.182 INFO:scrape:suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/snaps-few-objects}'] 2023-06-15T14:23:31.182 INFO:scrape: 2023-06-15T14:23:31.182 INFO:scrape:Failure: Command failed (workunit test rados/test.sh) on smithi043 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0b6ad8d73cfd5b848aa44abc5de6e89e15c19b20 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test.sh' 2023-06-15T14:23:31.183 INFO:scrape:1 jobs: ['7304969'] 2023-06-15T14:23:31.183 INFO:scrape:suites: ['clusters/fixed-2', 'crimson-rados/basic/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rados_api_tests}'] 2023-06-15T14:23:31.183 INFO:scrape: 2023-06-15T14:23:31.184 INFO:scrape:Dead: b'2023-06-15T14:23:24.964 INFO:teuthology.orchestra.run.smithi006.stderr:2023-06-15T14:23:24.959+0000 7f2cd5bc3700 -1 WARNING: the following dangerous and experimental features are enabled: crimson' 2023-06-15T14:23:31.184 INFO:scrape:1 jobs: ['7304985'] 2023-06-15T14:23:31.185 INFO:scrape:suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/snaps-few-objects-localized}'] 2023-06-15T14:23:31.185 INFO:scrape: 2023-06-15T14:23:31.186 INFO:teuthology.results:Sending results to yingxin.cheng@intel.com: Test Run: yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi ================================================================= info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/ logs: http://qa-proxy.ceph.com/teuthology/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/ failed: 3 dead: 5 running: 3 waiting: 0 queued: 0 passed: 9 Fail ================================================================= [7304969] crimson-rados/basic/{centos_latest clusters/fixed-2 crimson_qa_overrides deploy/ceph tasks/rados_api_tests} ----------------------------------------------------------------- time: 00:32:20 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304969/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304969/ sentry: https://sentry.ceph.com/organizations/ceph/?query=1420240cc4294fefa0fbdbfcc7e2d007 Command failed (workunit test rados/test.sh) on smithi043 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0b6ad8d73cfd5b848aa44abc5de6e89e15c19b20 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph- coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test.sh' [7304970] crimson-rados/rbd/{centos_latest clusters/fixed-1 crimson_qa_overrides deploy/ceph tasks/rbd_api_tests} ----------------------------------------------------------------- time: 03:15:31 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304970/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304970/ sentry: https://sentry.ceph.com/organizations/ceph/?query=f4424fa617d44effaa1ae07fb5b14d1e Command failed (workunit test rbd/crimson/test_crimson_librbd.sh) on smithi112 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0b6ad8d73cfd5b848aa44abc5de6e89e15c19b20 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 CRIMSON_COMPAT=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/crimson/test_crims on_librbd.sh' [7304984] crimson-rados/rbd/{centos_latest clusters/fixed-1 crimson_qa_overrides deploy/ceph tasks/rbd_python_api_tests} ----------------------------------------------------------------- time: 03:15:23 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304984/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304984/ sentry: https://sentry.ceph.com/organizations/ceph/?query=e0755ba4640048688c8661d3a879d1f4 Command failed (workunit test rbd/test_librbd_python.sh) on smithi193 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0b6ad8d73cfd5b848aa44abc5de6e89e15c19b20 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 CRIMSON_COMPAT=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd_python.sh --eval-attr \'not (SKIP_IF_CRIMSON)\'' Dead ================================================================= [7304971] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/admin_socket_objecter_requests} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304971/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304971/ hit max job timeout [7304974] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/radosbench-high-concurrency} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304974/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304974/ hit max job timeout [7304978] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/small-objects-balanced} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304978/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304978/ hit max job timeout [7304979] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/small-objects-localized} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304979/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304979/ hit max job timeout [7304981] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/small-objects} ----------------------------------------------------------------- time: 00:00:00 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304981/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304981/ hit max job timeout Running ================================================================= [7304982] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/snaps-few-objects-balanced} info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304982/ [7304985] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/snaps-few-objects-localized} info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304985/ [7304986] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/snaps-few-objects} info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304986/ Pass ================================================================= [7304972] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/pool-snaps-few-objects} time: 00:35:16 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304972/ [7304973] crimson-rados/rbd/{centos_latest clusters/fixed-1 crimson_qa_overrides deploy/ceph tasks/rbd_api_tests_old_format} time: 00:26:00 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304973/ [7304975] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/radosbench} time: 00:34:01 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304975/ [7304976] crimson-rados/basic/{centos_latest clusters/fixed-2 crimson_qa_overrides deploy/ceph tasks/rados_python} time: 00:23:12 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304976/ [7304977] crimson-rados/rbd/{centos_latest clusters/fixed-1 crimson_qa_overrides deploy/ceph tasks/rbd_cls_tests} time: 00:24:07 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304977/ [7304980] crimson-rados/rbd/{centos_latest clusters/fixed-1 crimson_qa_overrides deploy/ceph tasks/rbd_lock_and_fence} time: 00:16:28 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304980/ [7304983] crimson-rados/basic/{centos_latest clusters/fixed-2 crimson_qa_overrides deploy/ceph tasks/readwrite} time: 00:18:28 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304983/ [7304987] crimson-rados/rbd/{centos_latest clusters/fixed-1 crimson_qa_overrides deploy/ceph tasks/rbd_python_api_tests_old_format} time: 00:31:05 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304987/ [7304988] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/write_fadvise_dontneed} time: 00:20:31 info: http://pulpito.ceph.com/yingxin-2023-06-15_01:31:37-crimson-rados-wip-yingxin-msgr-multi-core-crimson-6-distro-default-smithi/7304988/