2022-04-28T21:54:25.070 INFO:root:teuthology version: 1.1.1.dev328+g004701de 2022-04-28T21:54:25.070 INFO:teuthology.results:subset: '111/120000' 2022-04-28T21:54:25.070 INFO:teuthology.results:seed: '230' 2022-04-28T23:06:36.191 INFO:root:teuthology version: 1.1.1.dev328+g004701de 2022-04-28T23:06:36.206 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2022-04-29T05:32:05.740 INFO:teuthology.results:Tests finished! gathering results... 2022-04-29T05:32:08.205 INFO:scrape:Found 25 jobs 2022-04-29T05:32:09.059 WARNING:scrape:Ignoring malparsed backtrace: ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f177a098488] , 2: (DeleteOp::_begin()+0x519) [0x55be60a122e9] 2022-04-29T05:32:09.060 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:28.946 INFO:teuthology.orchestra.run.smithi114.stdout: "initiated_at": "2022-04-28T23:32:33.899324+0000", , 2022-04-28T23:33:28.946 INFO:teuthology.orchestra.run.smithi114.stdout: "age": 55.027633780999999, , 2022-04-28T23:33:28.947 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.113241122, 2022-04-29T05:32:09.060 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:28.964 INFO:teuthology.orchestra.run.smithi114.stdout: "description": "MOSDPGPush(3.cs0 487/479 [PushOp(3:30f1babd:::smithi05083516-19:10e, version: 346'425, data_included: [1048576~524288], data_size: 524288, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(3:30f1babd:::smithi05083516-19:10e@346'425, size: 6553600, copy_subset: [], clone_subset: {}, snapset: 152=[]:{9d=[9d],db=[db,d8,d7,d4,d0,ce,c9,c6,c2,b5,ab,a5],e1=[e1,de],fd=[fd,fc,f9,f6,ef,ea,e8],10e=[10e,10d,10a,102],149=[148,147,146,144,13f],14c=[14c],152=[152,14f]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:3145728, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:2097152, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", , 2022-04-28T23:33:28.964 INFO:teuthology.orchestra.run.smithi114.stdout: "initiated_at": "2022-04-28T23:32:34.530907+0000", , 2022-04-28T23:33:28.964 INFO:teuthology.orchestra.run.smithi114.stdout: "age": 54.396050985999999, 2022-04-29T05:32:09.060 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:28.980 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:28.980 INFO:teuthology.orchestra.run.smithi114.stdout: "description": "MOSDPGPush(3.cs0 487/479 [PushOp(3:3ad70f12:::smithi05083516-47:116, version: 350'429, data_included: [1572864~524288], data_size: 524288, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(3:3ad70f12:::smithi05083516-47:116@350'429, size: 6234112, copy_subset: [], clone_subset: {}, snapset: 14c=[]:{f9=[f9,f6,f2,ef,ea,e8,e1,d8,b5],10a=[10a,108,102],116=[116,115,114,10e,10d],117=[117],11f=[11f],133=[133,132,131],14c=[14c,148,147,146,144,13f]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:3145728, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", , 2022-04-28T23:33:28.980 INFO:teuthology.orchestra.run.smithi114.stdout: "initiated_at": "2022-04-28T23:32:34.532933+0000", 2022-04-29T05:32:09.061 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.001 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.001 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:29.001 INFO:teuthology.orchestra.run.smithi114.stdout: "description": "MOSDPGPush(3.1s0 487/479 [PushOp(3:87764b60:::smithi05083516-9:head, version: 433'374, data_included: [2621440~524288], data_size: 524288, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(3:87764b60:::smithi05083516-9:head@433'374, size: 10469376, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:6291456, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:5242880, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", 2022-04-29T05:32:09.061 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.017 INFO:teuthology.orchestra.run.smithi114.stdout: } , 2022-04-28T23:33:29.017 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.021 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.061 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.034 INFO:teuthology.orchestra.run.smithi114.stdout: ] , 2022-04-28T23:33:29.034 INFO:teuthology.orchestra.run.smithi114.stdout: } , 2022-04-28T23:33:29.034 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.061 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.055 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.055 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:29.055 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "started", 2022-04-29T05:32:09.062 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.068 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 6.1650000000000003e-06 , 2022-04-28T23:33:29.068 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.068 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.062 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.079 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:32:48.971585+0000", , 2022-04-28T23:33:29.079 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 1.0903e-05 , 2022-04-28T23:33:29.079 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.062 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.089 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "header_read", , 2022-04-28T23:33:29.089 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:33:25.890932+0000", , 2022-04-28T23:33:29.089 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 9.5109999999999992e-06 2022-04-29T05:32:09.063 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.366 INFO:teuthology.orchestra.run.smithi050.stdout: "ops": [], , 2022-04-28T23:33:29.367 INFO:teuthology.orchestra.run.smithi050.stdout: "complaint_time": 30, , 2022-04-28T23:33:29.367 INFO:teuthology.orchestra.run.smithi050.stdout: "num_blocked_ops": 0 2022-04-29T05:32:09.064 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.585 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "all_read", , 2022-04-28T23:33:29.585 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:28:53.672899+0000", , 2022-04-28T23:33:29.585 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 9.1260000000000004e-06 2022-04-29T05:32:09.064 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.594 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.595 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "all_read", , 2022-04-28T23:33:29.595 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:28:58.111971+0000", 2022-04-29T05:32:09.064 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.604 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.604 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.604 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "all_read", 2022-04-29T05:32:09.064 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.616 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 1.2391e-05 , 2022-04-28T23:33:29.617 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.617 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.065 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.630 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:03.102477+0000", , 2022-04-28T23:33:29.630 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 1.6353000000000001e-05 , 2022-04-28T23:33:29.630 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.065 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.641 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:29.641 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:07.638944+0000", , 2022-04-28T23:33:29.641 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 8.2570000000000007e-06 2022-04-29T05:32:09.065 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.664 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.664 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:29.664 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:08.121352+0000", 2022-04-29T05:32:09.065 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.679 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.679 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.679 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", 2022-04-29T05:32:09.066 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.689 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967295.999999 , 2022-04-28T23:33:29.689 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.689 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.066 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.699 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:17.128118+0000", , 2022-04-28T23:33:29.699 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967295.999999 , 2022-04-28T23:33:29.700 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.066 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.093 INFO:teuthology.orchestra.run.smithi114.stdout: "client_info": { , 2022-04-28T23:33:30.093 INFO:teuthology.orchestra.run.smithi114.stdout: "client": "client.4527", , 2022-04-28T23:33:30.093 INFO:teuthology.orchestra.run.smithi114.stdout: "client_addr": "172.21.15.50:0/2700292602", 2022-04-29T05:32:09.067 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.102 INFO:teuthology.orchestra.run.smithi114.stdout: "events": [ , 2022-04-28T23:33:30.102 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.102 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "initiated", 2022-04-29T05:32:09.067 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.114 INFO:teuthology.orchestra.run.smithi114.stdout: "events": [ , 2022-04-28T23:33:30.114 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.114 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "initiated", 2022-04-29T05:32:09.067 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.126 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.126 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.126 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "throttled", 2022-04-29T05:32:09.067 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.136 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.136 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.136 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "header_read", 2022-04-29T05:32:09.068 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.150 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.151 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.151 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "queued_for_pg", 2022-04-29T05:32:09.068 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.162 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.162 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.163 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "reached_pg", 2022-04-29T05:32:09.068 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.174 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.174 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.175 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "waiting for rw locks", 2022-04-29T05:32:09.069 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.189 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.190 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.190 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "commit_sent", 2022-04-29T05:32:09.069 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.201 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 2.587e-06 , 2022-04-28T23:33:30.201 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.201 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.069 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.215 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.013865809 , 2022-04-28T23:33:30.215 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.215 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.070 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.225 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:31:21.763006+0000", , 2022-04-28T23:33:30.225 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0 , 2022-04-28T23:33:30.225 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.070 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.234 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:31:21.763144+0000", , 2022-04-28T23:33:30.235 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 6.4999999999999996e-06 , 2022-04-28T23:33:30.235 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.070 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.244 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:31:34.033010+0000", , 2022-04-28T23:33:30.245 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.28430043799999999 , 2022-04-28T23:33:30.245 INFO:teuthology.orchestra.run.smithi114.stdout: } 2022-04-29T05:32:09.070 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.254 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "done", , 2022-04-28T23:33:30.254 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:32:10.985951+0000", , 2022-04-28T23:33:30.255 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.299534256 2022-04-29T05:32:09.071 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.277 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.277 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "done", , 2022-04-28T23:33:30.277 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:32:25.297717+0000", 2022-04-29T05:32:09.071 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.681 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:30.681 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:33.470339+0000", , 2022-04-28T23:33:30.682 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0 2022-04-29T05:32:09.072 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.692 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.692 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:30.692 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:33.471283+0000", 2022-04-29T05:32:09.072 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.701 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.701 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.702 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", 2022-04-29T05:32:09.072 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.711 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0 , 2022-04-28T23:33:30.711 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.711 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.073 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.720 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:34.889003+0000", , 2022-04-28T23:33:30.720 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967296 , 2022-04-28T23:33:30.721 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.073 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.730 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "throttled", , 2022-04-28T23:33:30.730 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:34.889068+0000", , 2022-04-28T23:33:30.730 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967295.9999995 2022-04-29T05:32:09.074 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.740 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.740 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "throttled", , 2022-04-28T23:33:30.740 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:34.896960+0000", 2022-04-29T05:32:09.074 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.755 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.755 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.755 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "throttled", 2022-04-29T05:32:09.075 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.765 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0 , 2022-04-28T23:33:30.765 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.765 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.075 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.774 INFO:teuthology.orchestra.run.smithi050.stdout: "type_data": { , 2022-04-28T23:33:30.774 INFO:teuthology.orchestra.run.smithi050.stdout: "flag_point": "reached pg", , 2022-04-28T23:33:30.774 INFO:teuthology.orchestra.run.smithi050.stdout: "client_info": { 2022-04-29T05:32:09.075 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.057 INFO:teuthology.orchestra.run.smithi050.stdout: "num_blocked_ops": 0 , 2022-04-28T23:33:31.057 INFO:teuthology.orchestra.run.smithi050.stdout:} , 2022-04-28T23:33:31.067 DEBUG:teuthology.orchestra.run.smithi050:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok dump_historic_ops 2022-04-29T05:32:09.076 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.203 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.203 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "done", , 2022-04-28T23:33:31.203 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:28.939440+0000", 2022-04-29T05:32:09.076 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.215 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.215 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.215 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "done", 2022-04-29T05:32:09.076 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.233 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.00091377199999999998 , 2022-04-28T23:33:31.233 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.233 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.077 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.254 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:29.981114+0000", , 2022-04-28T23:33:31.254 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.00092280200000000002 , 2022-04-28T23:33:31.255 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.077 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.271 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "reached_pg", , 2022-04-28T23:33:31.272 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.191739+0000", , 2022-04-28T23:33:31.272 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.0011299890000000001 2022-04-29T05:32:09.099 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.281 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.281 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "reached_pg", , 2022-04-28T23:33:31.281 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.428968+0000", 2022-04-29T05:32:09.100 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.292 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.292 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.292 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "reached_pg", 2022-04-29T05:32:09.100 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.304 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 5.1112000000000001e-05 , 2022-04-28T23:33:31.304 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.305 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.100 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.314 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.969892+0000", , 2022-04-28T23:33:31.314 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.0031888440000000001 , 2022-04-28T23:33:31.314 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.100 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.325 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "queued_for_pg", , 2022-04-28T23:33:31.325 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.970391+0000", , 2022-04-28T23:33:31.325 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.102787499 2022-04-29T05:32:09.526 WARNING:scrape:Ignoring malparsed backtrace: ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f177a098488] , 2: (DeleteOp::_begin()+0x519) [0x55be60a122e9] 2022-04-29T05:32:09.527 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:28.946 INFO:teuthology.orchestra.run.smithi114.stdout: "initiated_at": "2022-04-28T23:32:33.899324+0000", , 2022-04-28T23:33:28.946 INFO:teuthology.orchestra.run.smithi114.stdout: "age": 55.027633780999999, , 2022-04-28T23:33:28.947 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.113241122, 2022-04-29T05:32:09.527 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:28.964 INFO:teuthology.orchestra.run.smithi114.stdout: "description": "MOSDPGPush(3.cs0 487/479 [PushOp(3:30f1babd:::smithi05083516-19:10e, version: 346'425, data_included: [1048576~524288], data_size: 524288, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(3:30f1babd:::smithi05083516-19:10e@346'425, size: 6553600, copy_subset: [], clone_subset: {}, snapset: 152=[]:{9d=[9d],db=[db,d8,d7,d4,d0,ce,c9,c6,c2,b5,ab,a5],e1=[e1,de],fd=[fd,fc,f9,f6,ef,ea,e8],10e=[10e,10d,10a,102],149=[148,147,146,144,13f],14c=[14c],152=[152,14f]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:3145728, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:2097152, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", , 2022-04-28T23:33:28.964 INFO:teuthology.orchestra.run.smithi114.stdout: "initiated_at": "2022-04-28T23:32:34.530907+0000", , 2022-04-28T23:33:28.964 INFO:teuthology.orchestra.run.smithi114.stdout: "age": 54.396050985999999, 2022-04-29T05:32:09.527 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:28.980 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:28.980 INFO:teuthology.orchestra.run.smithi114.stdout: "description": "MOSDPGPush(3.cs0 487/479 [PushOp(3:3ad70f12:::smithi05083516-47:116, version: 350'429, data_included: [1572864~524288], data_size: 524288, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(3:3ad70f12:::smithi05083516-47:116@350'429, size: 6234112, copy_subset: [], clone_subset: {}, snapset: 14c=[]:{f9=[f9,f6,f2,ef,ea,e8,e1,d8,b5],10a=[10a,108,102],116=[116,115,114,10e,10d],117=[117],11f=[11f],133=[133,132,131],14c=[14c,148,147,146,144,13f]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:3145728, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", , 2022-04-28T23:33:28.980 INFO:teuthology.orchestra.run.smithi114.stdout: "initiated_at": "2022-04-28T23:32:34.532933+0000", 2022-04-29T05:32:09.527 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.001 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.001 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:29.001 INFO:teuthology.orchestra.run.smithi114.stdout: "description": "MOSDPGPush(3.1s0 487/479 [PushOp(3:87764b60:::smithi05083516-9:head, version: 433'374, data_included: [2621440~524288], data_size: 524288, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(3:87764b60:::smithi05083516-9:head@433'374, size: 10469376, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:6291456, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:5242880, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", 2022-04-29T05:32:09.528 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.017 INFO:teuthology.orchestra.run.smithi114.stdout: } , 2022-04-28T23:33:29.017 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.021 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.528 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.034 INFO:teuthology.orchestra.run.smithi114.stdout: ] , 2022-04-28T23:33:29.034 INFO:teuthology.orchestra.run.smithi114.stdout: } , 2022-04-28T23:33:29.034 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.528 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.055 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.055 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:29.055 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "started", 2022-04-29T05:32:09.528 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.068 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 6.1650000000000003e-06 , 2022-04-28T23:33:29.068 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.068 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.529 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.079 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:32:48.971585+0000", , 2022-04-28T23:33:29.079 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 1.0903e-05 , 2022-04-28T23:33:29.079 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.529 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.089 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "header_read", , 2022-04-28T23:33:29.089 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:33:25.890932+0000", , 2022-04-28T23:33:29.089 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 9.5109999999999992e-06 2022-04-29T05:32:09.529 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.366 INFO:teuthology.orchestra.run.smithi050.stdout: "ops": [], , 2022-04-28T23:33:29.367 INFO:teuthology.orchestra.run.smithi050.stdout: "complaint_time": 30, , 2022-04-28T23:33:29.367 INFO:teuthology.orchestra.run.smithi050.stdout: "num_blocked_ops": 0 2022-04-29T05:32:09.529 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.585 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "all_read", , 2022-04-28T23:33:29.585 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:28:53.672899+0000", , 2022-04-28T23:33:29.585 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 9.1260000000000004e-06 2022-04-29T05:32:09.530 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.594 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.595 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "all_read", , 2022-04-28T23:33:29.595 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:28:58.111971+0000", 2022-04-29T05:32:09.530 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.604 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.604 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.604 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "all_read", 2022-04-29T05:32:09.530 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.616 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 1.2391e-05 , 2022-04-28T23:33:29.617 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.617 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.530 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.630 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:03.102477+0000", , 2022-04-28T23:33:29.630 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 1.6353000000000001e-05 , 2022-04-28T23:33:29.630 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.531 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.641 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:29.641 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:07.638944+0000", , 2022-04-28T23:33:29.641 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 8.2570000000000007e-06 2022-04-29T05:32:09.531 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.664 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.664 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:29.664 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:08.121352+0000", 2022-04-29T05:32:09.531 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.679 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.679 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.679 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", 2022-04-29T05:32:09.531 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.689 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967295.999999 , 2022-04-28T23:33:29.689 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.689 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.531 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.699 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:17.128118+0000", , 2022-04-28T23:33:29.699 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967295.999999 , 2022-04-28T23:33:29.700 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.532 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.093 INFO:teuthology.orchestra.run.smithi114.stdout: "client_info": { , 2022-04-28T23:33:30.093 INFO:teuthology.orchestra.run.smithi114.stdout: "client": "client.4527", , 2022-04-28T23:33:30.093 INFO:teuthology.orchestra.run.smithi114.stdout: "client_addr": "172.21.15.50:0/2700292602", 2022-04-29T05:32:09.532 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.102 INFO:teuthology.orchestra.run.smithi114.stdout: "events": [ , 2022-04-28T23:33:30.102 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.102 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "initiated", 2022-04-29T05:32:09.532 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.114 INFO:teuthology.orchestra.run.smithi114.stdout: "events": [ , 2022-04-28T23:33:30.114 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.114 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "initiated", 2022-04-29T05:32:09.532 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.126 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.126 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.126 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "throttled", 2022-04-29T05:32:09.533 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.136 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.136 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.136 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "header_read", 2022-04-29T05:32:09.533 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.150 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.151 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.151 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "queued_for_pg", 2022-04-29T05:32:09.533 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.162 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.162 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.163 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "reached_pg", 2022-04-29T05:32:09.533 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.174 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.174 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.175 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "waiting for rw locks", 2022-04-29T05:32:09.534 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.189 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.190 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.190 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "commit_sent", 2022-04-29T05:32:09.534 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.201 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 2.587e-06 , 2022-04-28T23:33:30.201 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.201 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.534 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.215 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.013865809 , 2022-04-28T23:33:30.215 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.215 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.534 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.225 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:31:21.763006+0000", , 2022-04-28T23:33:30.225 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0 , 2022-04-28T23:33:30.225 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.535 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.234 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:31:21.763144+0000", , 2022-04-28T23:33:30.235 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 6.4999999999999996e-06 , 2022-04-28T23:33:30.235 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.535 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.244 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:31:34.033010+0000", , 2022-04-28T23:33:30.245 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.28430043799999999 , 2022-04-28T23:33:30.245 INFO:teuthology.orchestra.run.smithi114.stdout: } 2022-04-29T05:32:09.535 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.254 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "done", , 2022-04-28T23:33:30.254 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:32:10.985951+0000", , 2022-04-28T23:33:30.255 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.299534256 2022-04-29T05:32:09.535 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.277 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.277 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "done", , 2022-04-28T23:33:30.277 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:32:25.297717+0000", 2022-04-29T05:32:09.536 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.681 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:30.681 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:33.470339+0000", , 2022-04-28T23:33:30.682 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0 2022-04-29T05:32:09.536 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.692 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.692 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:30.692 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:33.471283+0000", 2022-04-29T05:32:09.536 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.701 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.701 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.702 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", 2022-04-29T05:32:09.536 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.711 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0 , 2022-04-28T23:33:30.711 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.711 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.536 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.720 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:34.889003+0000", , 2022-04-28T23:33:30.720 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967296 , 2022-04-28T23:33:30.721 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.537 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.730 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "throttled", , 2022-04-28T23:33:30.730 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:34.889068+0000", , 2022-04-28T23:33:30.730 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967295.9999995 2022-04-29T05:32:09.537 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.740 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.740 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "throttled", , 2022-04-28T23:33:30.740 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:34.896960+0000", 2022-04-29T05:32:09.537 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.755 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.755 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.755 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "throttled", 2022-04-29T05:32:09.538 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.765 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0 , 2022-04-28T23:33:30.765 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.765 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.538 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.774 INFO:teuthology.orchestra.run.smithi050.stdout: "type_data": { , 2022-04-28T23:33:30.774 INFO:teuthology.orchestra.run.smithi050.stdout: "flag_point": "reached pg", , 2022-04-28T23:33:30.774 INFO:teuthology.orchestra.run.smithi050.stdout: "client_info": { 2022-04-29T05:32:09.539 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.057 INFO:teuthology.orchestra.run.smithi050.stdout: "num_blocked_ops": 0 , 2022-04-28T23:33:31.057 INFO:teuthology.orchestra.run.smithi050.stdout:} , 2022-04-28T23:33:31.067 DEBUG:teuthology.orchestra.run.smithi050:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok dump_historic_ops 2022-04-29T05:32:09.539 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.203 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.203 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "done", , 2022-04-28T23:33:31.203 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:28.939440+0000", 2022-04-29T05:32:09.539 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.215 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.215 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.215 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "done", 2022-04-29T05:32:09.539 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.233 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.00091377199999999998 , 2022-04-28T23:33:31.233 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.233 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.540 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.254 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:29.981114+0000", , 2022-04-28T23:33:31.254 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.00092280200000000002 , 2022-04-28T23:33:31.255 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.545 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.271 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "reached_pg", , 2022-04-28T23:33:31.272 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.191739+0000", , 2022-04-28T23:33:31.272 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.0011299890000000001 2022-04-29T05:32:09.545 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.281 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.281 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "reached_pg", , 2022-04-28T23:33:31.281 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.428968+0000", 2022-04-29T05:32:09.545 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.292 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.292 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.292 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "reached_pg", 2022-04-29T05:32:09.546 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.304 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 5.1112000000000001e-05 , 2022-04-28T23:33:31.304 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.305 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.546 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.314 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.969892+0000", , 2022-04-28T23:33:31.314 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.0031888440000000001 , 2022-04-28T23:33:31.314 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.546 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.325 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "queued_for_pg", , 2022-04-28T23:33:31.325 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.970391+0000", , 2022-04-28T23:33:31.325 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.102787499 2022-04-29T05:32:09.955 WARNING:scrape:Ignoring malparsed backtrace: ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f177a098488] , 2: (DeleteOp::_begin()+0x519) [0x55be60a122e9] 2022-04-29T05:32:09.956 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:28.946 INFO:teuthology.orchestra.run.smithi114.stdout: "initiated_at": "2022-04-28T23:32:33.899324+0000", , 2022-04-28T23:33:28.946 INFO:teuthology.orchestra.run.smithi114.stdout: "age": 55.027633780999999, , 2022-04-28T23:33:28.947 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.113241122, 2022-04-29T05:32:09.956 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:28.964 INFO:teuthology.orchestra.run.smithi114.stdout: "description": "MOSDPGPush(3.cs0 487/479 [PushOp(3:30f1babd:::smithi05083516-19:10e, version: 346'425, data_included: [1048576~524288], data_size: 524288, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(3:30f1babd:::smithi05083516-19:10e@346'425, size: 6553600, copy_subset: [], clone_subset: {}, snapset: 152=[]:{9d=[9d],db=[db,d8,d7,d4,d0,ce,c9,c6,c2,b5,ab,a5],e1=[e1,de],fd=[fd,fc,f9,f6,ef,ea,e8],10e=[10e,10d,10a,102],149=[148,147,146,144,13f],14c=[14c],152=[152,14f]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:3145728, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:2097152, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", , 2022-04-28T23:33:28.964 INFO:teuthology.orchestra.run.smithi114.stdout: "initiated_at": "2022-04-28T23:32:34.530907+0000", , 2022-04-28T23:33:28.964 INFO:teuthology.orchestra.run.smithi114.stdout: "age": 54.396050985999999, 2022-04-29T05:32:09.956 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:28.980 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:28.980 INFO:teuthology.orchestra.run.smithi114.stdout: "description": "MOSDPGPush(3.cs0 487/479 [PushOp(3:3ad70f12:::smithi05083516-47:116, version: 350'429, data_included: [1572864~524288], data_size: 524288, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(3:3ad70f12:::smithi05083516-47:116@350'429, size: 6234112, copy_subset: [], clone_subset: {}, snapset: 14c=[]:{f9=[f9,f6,f2,ef,ea,e8,e1,d8,b5],10a=[10a,108,102],116=[116,115,114,10e,10d],117=[117],11f=[11f],133=[133,132,131],14c=[14c,148,147,146,144,13f]}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:3145728, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", , 2022-04-28T23:33:28.980 INFO:teuthology.orchestra.run.smithi114.stdout: "initiated_at": "2022-04-28T23:32:34.532933+0000", 2022-04-29T05:32:09.956 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.001 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.001 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:29.001 INFO:teuthology.orchestra.run.smithi114.stdout: "description": "MOSDPGPush(3.1s0 487/479 [PushOp(3:87764b60:::smithi05083516-9:head, version: 433'374, data_included: [2621440~524288], data_size: 524288, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(3:87764b60:::smithi05083516-9:head@433'374, size: 10469376, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 1), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:6291456, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:5242880, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", 2022-04-29T05:32:09.957 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.017 INFO:teuthology.orchestra.run.smithi114.stdout: } , 2022-04-28T23:33:29.017 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.021 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.957 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.034 INFO:teuthology.orchestra.run.smithi114.stdout: ] , 2022-04-28T23:33:29.034 INFO:teuthology.orchestra.run.smithi114.stdout: } , 2022-04-28T23:33:29.034 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.957 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.055 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.055 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:29.055 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "started", 2022-04-29T05:32:09.957 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.068 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 6.1650000000000003e-06 , 2022-04-28T23:33:29.068 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:29.068 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.958 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.079 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:32:48.971585+0000", , 2022-04-28T23:33:29.079 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 1.0903e-05 , 2022-04-28T23:33:29.079 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.958 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.089 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "header_read", , 2022-04-28T23:33:29.089 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:33:25.890932+0000", , 2022-04-28T23:33:29.089 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 9.5109999999999992e-06 2022-04-29T05:32:09.958 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.366 INFO:teuthology.orchestra.run.smithi050.stdout: "ops": [], , 2022-04-28T23:33:29.367 INFO:teuthology.orchestra.run.smithi050.stdout: "complaint_time": 30, , 2022-04-28T23:33:29.367 INFO:teuthology.orchestra.run.smithi050.stdout: "num_blocked_ops": 0 2022-04-29T05:32:09.958 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.585 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "all_read", , 2022-04-28T23:33:29.585 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:28:53.672899+0000", , 2022-04-28T23:33:29.585 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 9.1260000000000004e-06 2022-04-29T05:32:09.959 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.594 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.595 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "all_read", , 2022-04-28T23:33:29.595 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:28:58.111971+0000", 2022-04-29T05:32:09.959 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.604 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.604 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.604 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "all_read", 2022-04-29T05:32:09.959 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.616 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 1.2391e-05 , 2022-04-28T23:33:29.617 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.617 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.959 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.630 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:03.102477+0000", , 2022-04-28T23:33:29.630 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 1.6353000000000001e-05 , 2022-04-28T23:33:29.630 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.960 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.641 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:29.641 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:07.638944+0000", , 2022-04-28T23:33:29.641 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 8.2570000000000007e-06 2022-04-29T05:32:09.960 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.664 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.664 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:29.664 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:08.121352+0000", 2022-04-29T05:32:09.960 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.679 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.679 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:29.679 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", 2022-04-29T05:32:09.960 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.689 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967295.999999 , 2022-04-28T23:33:29.689 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:29.689 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.961 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:29.699 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:29:17.128118+0000", , 2022-04-28T23:33:29.699 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967295.999999 , 2022-04-28T23:33:29.700 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.961 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.093 INFO:teuthology.orchestra.run.smithi114.stdout: "client_info": { , 2022-04-28T23:33:30.093 INFO:teuthology.orchestra.run.smithi114.stdout: "client": "client.4527", , 2022-04-28T23:33:30.093 INFO:teuthology.orchestra.run.smithi114.stdout: "client_addr": "172.21.15.50:0/2700292602", 2022-04-29T05:32:09.961 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.102 INFO:teuthology.orchestra.run.smithi114.stdout: "events": [ , 2022-04-28T23:33:30.102 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.102 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "initiated", 2022-04-29T05:32:09.962 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.114 INFO:teuthology.orchestra.run.smithi114.stdout: "events": [ , 2022-04-28T23:33:30.114 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.114 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "initiated", 2022-04-29T05:32:09.962 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.126 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.126 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.126 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "throttled", 2022-04-29T05:32:09.962 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.136 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.136 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.136 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "header_read", 2022-04-29T05:32:09.962 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.150 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.151 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.151 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "queued_for_pg", 2022-04-29T05:32:09.963 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.162 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.162 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.163 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "reached_pg", 2022-04-29T05:32:09.963 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.174 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.174 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.175 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "waiting for rw locks", 2022-04-29T05:32:09.963 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.189 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.190 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.190 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "commit_sent", 2022-04-29T05:32:09.964 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.201 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 2.587e-06 , 2022-04-28T23:33:30.201 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.201 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.964 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.215 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.013865809 , 2022-04-28T23:33:30.215 INFO:teuthology.orchestra.run.smithi114.stdout: }, , 2022-04-28T23:33:30.215 INFO:teuthology.orchestra.run.smithi114.stdout: { 2022-04-29T05:32:09.964 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.225 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:31:21.763006+0000", , 2022-04-28T23:33:30.225 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0 , 2022-04-28T23:33:30.225 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.964 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.234 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:31:21.763144+0000", , 2022-04-28T23:33:30.235 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 6.4999999999999996e-06 , 2022-04-28T23:33:30.235 INFO:teuthology.orchestra.run.smithi114.stdout: }, 2022-04-29T05:32:09.965 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.244 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:31:34.033010+0000", , 2022-04-28T23:33:30.245 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.28430043799999999 , 2022-04-28T23:33:30.245 INFO:teuthology.orchestra.run.smithi114.stdout: } 2022-04-29T05:32:09.965 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.254 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "done", , 2022-04-28T23:33:30.254 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:32:10.985951+0000", , 2022-04-28T23:33:30.255 INFO:teuthology.orchestra.run.smithi114.stdout: "duration": 0.299534256 2022-04-29T05:32:09.965 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.277 INFO:teuthology.orchestra.run.smithi114.stdout: { , 2022-04-28T23:33:30.277 INFO:teuthology.orchestra.run.smithi114.stdout: "event": "done", , 2022-04-28T23:33:30.277 INFO:teuthology.orchestra.run.smithi114.stdout: "time": "2022-04-28T23:32:25.297717+0000", 2022-04-29T05:32:09.965 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.681 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:30.681 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:33.470339+0000", , 2022-04-28T23:33:30.682 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0 2022-04-29T05:32:09.966 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.692 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.692 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", , 2022-04-28T23:33:30.692 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:33.471283+0000", 2022-04-29T05:32:09.966 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.701 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.701 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.702 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "header_read", 2022-04-29T05:32:09.966 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.711 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0 , 2022-04-28T23:33:30.711 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.711 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.966 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.720 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:34.889003+0000", , 2022-04-28T23:33:30.720 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967296 , 2022-04-28T23:33:30.721 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.966 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.730 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "throttled", , 2022-04-28T23:33:30.730 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:34.889068+0000", , 2022-04-28T23:33:30.730 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 4294967295.9999995 2022-04-29T05:32:09.973 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.740 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.740 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "throttled", , 2022-04-28T23:33:30.740 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:32:34.896960+0000", 2022-04-29T05:32:09.973 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.755 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.755 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:30.755 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "throttled", 2022-04-29T05:32:09.974 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.765 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0 , 2022-04-28T23:33:30.765 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:30.765 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.974 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:30.774 INFO:teuthology.orchestra.run.smithi050.stdout: "type_data": { , 2022-04-28T23:33:30.774 INFO:teuthology.orchestra.run.smithi050.stdout: "flag_point": "reached pg", , 2022-04-28T23:33:30.774 INFO:teuthology.orchestra.run.smithi050.stdout: "client_info": { 2022-04-29T05:32:09.975 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.057 INFO:teuthology.orchestra.run.smithi050.stdout: "num_blocked_ops": 0 , 2022-04-28T23:33:31.057 INFO:teuthology.orchestra.run.smithi050.stdout:} , 2022-04-28T23:33:31.067 DEBUG:teuthology.orchestra.run.smithi050:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok dump_historic_ops 2022-04-29T05:32:09.975 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.203 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.203 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "done", , 2022-04-28T23:33:31.203 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:28.939440+0000", 2022-04-29T05:32:09.975 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.215 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.215 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.215 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "done", 2022-04-29T05:32:09.976 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.233 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.00091377199999999998 , 2022-04-28T23:33:31.233 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.233 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.976 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.254 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:29.981114+0000", , 2022-04-28T23:33:31.254 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.00092280200000000002 , 2022-04-28T23:33:31.255 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.976 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.271 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "reached_pg", , 2022-04-28T23:33:31.272 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.191739+0000", , 2022-04-28T23:33:31.272 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.0011299890000000001 2022-04-29T05:32:09.976 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.281 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.281 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "reached_pg", , 2022-04-28T23:33:31.281 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.428968+0000", 2022-04-29T05:32:09.977 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.292 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.292 INFO:teuthology.orchestra.run.smithi050.stdout: { , 2022-04-28T23:33:31.292 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "reached_pg", 2022-04-29T05:32:09.978 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.304 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 5.1112000000000001e-05 , 2022-04-28T23:33:31.304 INFO:teuthology.orchestra.run.smithi050.stdout: }, , 2022-04-28T23:33:31.305 INFO:teuthology.orchestra.run.smithi050.stdout: { 2022-04-29T05:32:09.980 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.314 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.969892+0000", , 2022-04-28T23:33:31.314 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.0031888440000000001 , 2022-04-28T23:33:31.314 INFO:teuthology.orchestra.run.smithi050.stdout: }, 2022-04-29T05:32:09.980 WARNING:scrape:Ignoring malparsed backtrace: 2022-04-28T23:33:31.325 INFO:teuthology.orchestra.run.smithi050.stdout: "event": "queued_for_pg", , 2022-04-28T23:33:31.325 INFO:teuthology.orchestra.run.smithi050.stdout: "time": "2022-04-28T23:33:30.970391+0000", , 2022-04-28T23:33:31.325 INFO:teuthology.orchestra.run.smithi050.stdout: "duration": 0.102787499 2022-04-29T05:32:09.981 INFO:scrape:Found 2 distinct failure reasons 2022-04-29T05:32:09.981 INFO:scrape:Dead: b'2022-04-29T05:27:51.551 DEBUG:teuthology.exit:Finished running handlers' 2022-04-29T05:32:09.981 INFO:scrape:2 jobs: ['6811867', '6811862'] 2022-04-29T05:32:09.981 INFO:scrape:suites intersection: ['ceph', 'clusters/{fixed-2', 'mon_election/classic', 'msgr-failures/osd-dispatch-delay', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'recovery-overrides/{default}', 'supported-random-distro$/{rhel_8}', 'thrashers/default', 'thrashosds-health', 'workloads/ec-rados-plugin=isa-k=2-m=1}'] 2022-04-29T05:32:09.981 INFO:scrape:suites union: ['ceph', 'clusters/{fixed-2', 'mon_election/classic', 'msgr-failures/osd-dispatch-delay', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'recovery-overrides/{default}', 'supported-random-distro$/{rhel_8}', 'thrashers/default', 'thrashosds-health', 'workloads/ec-rados-plugin=isa-k=2-m=1}'] 2022-04-29T05:32:09.982 INFO:scrape: 2022-04-29T05:32:09.982 INFO:scrape:Crash: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --no-omap --ec-pool --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 0 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --op setattr 25 --op rmattr 25 --op copy_from 50 --op append 100 --pool unique_pool_0' 2022-04-29T05:32:09.982 INFO:scrape:ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable) 1: /lib64/libpthread.so.0(+0x12b20) [0x7f05a9023b20] 2: gsignal() 3: abort() 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x1b6) [0x55601a29511b] 5: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x1507) [0x55601a8dfad7] 6: (BlueStore::queue_transactions(boost::intrusive_ptr&, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x316) [0x55601a8fbae6] 7: (non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector >&, boost::intrusive_ptr)+0x58) [0x55601a53fc58] 8: (ECBackend::handle_sub_write(pg_shard_t, boost::intrusive_ptr, ECSubWrite&, ZTracer::Trace const&)+0xd1d) [0x55601a7591fd] 9: (ECBackend::_handle_message(boost::intrusive_ptr)+0x306) [0x55601a76fc06] 10: (PGBackend::handle_message(boost::intrusive_ptr)+0x52) [0x55601a573132] 11: (PrimaryLogPG::do_request(boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x5de) [0x55601a516a6e] 12: (OSD::dequeue_op(boost::intrusive_ptr, boost::intrusive_ptr, ThreadPool::TPHandle&)+0x309) [0x55601a39e149] 13: (ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x68) [0x55601a5fd258] 14: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xc28) [0x55601a3bb648] 15: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x55601aa38664] 16: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x55601aa3b544] 17: /lib64/libpthread.so.0(+0x814a) [0x7f05a901914a] 18: clone() 2022-04-29T05:32:09.982 INFO:scrape:1 jobs: ['6811874'] 2022-04-29T05:32:09.982 INFO:scrape:suites: ['ceph', 'clusters/{fixed-2', 'mon_election/classic', 'msgr-failures/osd-dispatch-delay', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'recovery-overrides/{default}', 'supported-random-distro$/{rhel_8}', 'thrashers/default', 'thrashosds-health', 'workloads/ec-rados-plugin=isa-k=2-m=1}'] 2022-04-29T05:32:09.982 INFO:scrape: