Found 378 jobs Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f60cc8c515a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55d61134dfd6] Ignoring malparsed backtrace: 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "op_commit", , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:36.033692+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:12.616 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:12.659 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.702 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.723 INFO:teuthology.orchestra.run.gibba027.stdout: } , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: ] , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:12.744 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0 , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 5.9333000000000003e-05 , 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.788 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "tid": 9733 , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:28.740029+0000", , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 1.3723000000000001e-05 , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.587417+0000", , 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.1121999999999997e-05 , 2021-01-28T21:30:12.852 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "header_read", , 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.121706+0000", , 2021-01-28T21:30:12.873 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.653e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:12.893 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:30:04.036293+0000", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.7120999999999999e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:13.253 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:13.301 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057574781999999998, , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 126.47379893, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057546552000000001, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for subops from 0", , 2021-01-28T21:30:13.351 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:27.384701+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:30.966124+0000", , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 102.286861452, , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.115346143, Ignoring malparsed backtrace: 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPushReply(3.d 47/45 [PushReplyOp(3:bd3332bf:::gibba02740590-173:head)])", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:31.905885+0000", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 101.347100532, Ignoring malparsed backtrace: 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPush(3.9 76/75 [PushOp(3:976daa4c:::gibba02740590-388:head, version: 25'85, data_included: [504309~539013,1796649~509563], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(3:976daa4c:::gibba02740590-388:head@25'85, size: 2504309, copy_subset: [0~2504309], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:2306212, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:29:29.152628+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.449 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.401611+0000", , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.6767999999999997e-05 , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.474 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.699165+0000", , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.29836768899999999 , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:48.844741+0000", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.00096275600000000005 Ignoring malparsed backtrace: 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "started", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:49.500590+0000", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 6.5687999999999999e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise , raise value.with_traceback(tb) , File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run Ignoring malparsed backtrace: 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.6723000000000005e-05 , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.918 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.15017624199999999 , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.983559+0000", , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00013104499999999999 , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", , 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.676623+0000", , 2021-01-28T21:30:13.964 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 Ignoring malparsed backtrace: 2021-01-28T21:30:13.985 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.682198+0000", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.7670000000000003e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.687855+0000", , 2021-01-28T21:30:14.008 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.3838000000000002e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.868039+0000", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.6422000000000003e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.869901+0000", , 2021-01-28T21:30:14.052 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.3876999999999998e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 49.654020881999998, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.191085802, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.097 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:24.415540+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.119 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.141 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.2329999999999997e-06 , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836246+0000", , 2021-01-28T21:30:14.508 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8579999999999999e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836932+0000", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.096930559999999999 Ignoring malparsed backtrace: 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.576 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 2171 Ignoring malparsed backtrace: 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "throttled", , 2021-01-28T21:30:14.600 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.935954+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.621 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.631600+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.635256+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.684 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00053292300000000003 , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.0094789959999999999 , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.768 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.1505000000000003e-05 , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 9665 , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:28.096642+0000", , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8930000000000004e-06 , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:39.086006+0000", , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00098902199999999999 , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:47.343564+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:15.224 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", Ignoring malparsed backtrace: 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.4930000000000001e-06 , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.001762855 , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.338 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.014717160999999999 , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:16.248449+0000", , 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:15.366 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:36.482680+0000", , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00091062800000000002 , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:58.200337+0000", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.53033678399999995 Ignoring malparsed backtrace: 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 41.744552429000002, , 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.017655751000000001, , 2021-01-28T21:30:15.432 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:33.514095+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.087655661999999995, , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:43.952803+0000", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.5390000000000002e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.040258+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.862 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for subops from 4", , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.151551+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.884 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.153686+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "done", , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.158101+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.930 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-01-28T21:30:15.977 INFO:teuthology.orchestra.run.gibba003.stdout: "description": "osd_op(client.4409.0:4502 3.6 3:606549d0:::gibba02740590-302:head [setxattr _header (56) in=7b,truncate 3664091] snapc 0=[] ondisk+write+known_if_redirected e36)", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "initiated_at": "2021-01-28T21:28:16.015999+0000", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 119.758439964, Ignoring malparsed backtrace: 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 1.5741000000000001e-05 , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:16.025 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.73173625200000003 , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.588895+0000", , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.072 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.297023+0000", , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.5664e-05 , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.668766+0000", , 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.37291091900000001 , 2021-01-28T21:30:16.097 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: reweighted osd.7 to 1 (10000) , 2021-01-28T21:30:29.060 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 7 1 , set osd.7 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: instructed osd(s) 2 to deep-scrub , 2021-01-28T21:30:38.281 INFO:tasks.ceph:Scrubbing osd.3 , 2021-01-28T21:30:38.282 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() Ignoring malparsed backtrace: 2021-01-28T21:31:24.207 DEBUG:teuthology.orchestra.run.gibba003:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2021-01-28T21:31:24.228 INFO:tasks.ceph:Compressing logs... , 2021-01-28T21:31:24.229 DEBUG:teuthology.orchestra.run.gibba003:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- Ignoring malparsed backtrace: 2021-01-28T21:31:38.162 DEBUG:teuthology.orchestra.run.gibba027:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . , 2021-01-28T21:32:00.691 INFO:teuthology.task.internal:Transferring binaries for coredumps... , 2021-01-28T21:32:00.720 ERROR:teuthology.run_tasks:Manager failed: internal.archive Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f60cc8c515a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55d61134dfd6] Ignoring malparsed backtrace: 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "op_commit", , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:36.033692+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:12.616 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:12.659 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.702 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.723 INFO:teuthology.orchestra.run.gibba027.stdout: } , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: ] , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:12.744 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0 , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 5.9333000000000003e-05 , 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.788 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "tid": 9733 , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:28.740029+0000", , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 1.3723000000000001e-05 , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.587417+0000", , 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.1121999999999997e-05 , 2021-01-28T21:30:12.852 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "header_read", , 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.121706+0000", , 2021-01-28T21:30:12.873 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.653e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:12.893 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:30:04.036293+0000", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.7120999999999999e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:13.253 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:13.301 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057574781999999998, , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 126.47379893, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057546552000000001, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for subops from 0", , 2021-01-28T21:30:13.351 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:27.384701+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:30.966124+0000", , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 102.286861452, , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.115346143, Ignoring malparsed backtrace: 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPushReply(3.d 47/45 [PushReplyOp(3:bd3332bf:::gibba02740590-173:head)])", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:31.905885+0000", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 101.347100532, Ignoring malparsed backtrace: 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPush(3.9 76/75 [PushOp(3:976daa4c:::gibba02740590-388:head, version: 25'85, data_included: [504309~539013,1796649~509563], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(3:976daa4c:::gibba02740590-388:head@25'85, size: 2504309, copy_subset: [0~2504309], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:2306212, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:29:29.152628+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.449 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.401611+0000", , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.6767999999999997e-05 , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.474 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.699165+0000", , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.29836768899999999 , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:48.844741+0000", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.00096275600000000005 Ignoring malparsed backtrace: 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "started", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:49.500590+0000", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 6.5687999999999999e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise , raise value.with_traceback(tb) , File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run Ignoring malparsed backtrace: 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.6723000000000005e-05 , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.918 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.15017624199999999 , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.983559+0000", , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00013104499999999999 , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", , 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.676623+0000", , 2021-01-28T21:30:13.964 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 Ignoring malparsed backtrace: 2021-01-28T21:30:13.985 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.682198+0000", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.7670000000000003e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.687855+0000", , 2021-01-28T21:30:14.008 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.3838000000000002e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.868039+0000", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.6422000000000003e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.869901+0000", , 2021-01-28T21:30:14.052 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.3876999999999998e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 49.654020881999998, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.191085802, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.097 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:24.415540+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.119 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.141 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.2329999999999997e-06 , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836246+0000", , 2021-01-28T21:30:14.508 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8579999999999999e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836932+0000", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.096930559999999999 Ignoring malparsed backtrace: 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.576 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 2171 Ignoring malparsed backtrace: 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "throttled", , 2021-01-28T21:30:14.600 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.935954+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.621 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.631600+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.635256+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.684 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00053292300000000003 , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.0094789959999999999 , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.768 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.1505000000000003e-05 , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 9665 , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:28.096642+0000", , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8930000000000004e-06 , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:39.086006+0000", , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00098902199999999999 , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:47.343564+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:15.224 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", Ignoring malparsed backtrace: 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.4930000000000001e-06 , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.001762855 , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.338 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.014717160999999999 , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:16.248449+0000", , 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:15.366 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:36.482680+0000", , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00091062800000000002 , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:58.200337+0000", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.53033678399999995 Ignoring malparsed backtrace: 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 41.744552429000002, , 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.017655751000000001, , 2021-01-28T21:30:15.432 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:33.514095+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.087655661999999995, , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:43.952803+0000", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.5390000000000002e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.040258+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.862 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for subops from 4", , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.151551+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.884 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.153686+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "done", , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.158101+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.930 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-01-28T21:30:15.977 INFO:teuthology.orchestra.run.gibba003.stdout: "description": "osd_op(client.4409.0:4502 3.6 3:606549d0:::gibba02740590-302:head [setxattr _header (56) in=7b,truncate 3664091] snapc 0=[] ondisk+write+known_if_redirected e36)", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "initiated_at": "2021-01-28T21:28:16.015999+0000", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 119.758439964, Ignoring malparsed backtrace: 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 1.5741000000000001e-05 , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:16.025 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.73173625200000003 , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.588895+0000", , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.072 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.297023+0000", , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.5664e-05 , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.668766+0000", , 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.37291091900000001 , 2021-01-28T21:30:16.097 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: reweighted osd.7 to 1 (10000) , 2021-01-28T21:30:29.060 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 7 1 , set osd.7 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: instructed osd(s) 2 to deep-scrub , 2021-01-28T21:30:38.281 INFO:tasks.ceph:Scrubbing osd.3 , 2021-01-28T21:30:38.282 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() Ignoring malparsed backtrace: 2021-01-28T21:31:24.207 DEBUG:teuthology.orchestra.run.gibba003:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2021-01-28T21:31:24.228 INFO:tasks.ceph:Compressing logs... , 2021-01-28T21:31:24.229 DEBUG:teuthology.orchestra.run.gibba003:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- Ignoring malparsed backtrace: 2021-01-28T21:31:38.162 DEBUG:teuthology.orchestra.run.gibba027:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . , 2021-01-28T21:32:00.691 INFO:teuthology.task.internal:Transferring binaries for coredumps... , 2021-01-28T21:32:00.720 ERROR:teuthology.run_tasks:Manager failed: internal.archive Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f60cc8c515a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55d61134dfd6] Ignoring malparsed backtrace: 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "op_commit", , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:36.033692+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:12.616 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:12.659 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.702 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.723 INFO:teuthology.orchestra.run.gibba027.stdout: } , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: ] , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:12.744 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0 , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 5.9333000000000003e-05 , 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.788 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "tid": 9733 , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:28.740029+0000", , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 1.3723000000000001e-05 , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.587417+0000", , 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.1121999999999997e-05 , 2021-01-28T21:30:12.852 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "header_read", , 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.121706+0000", , 2021-01-28T21:30:12.873 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.653e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:12.893 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:30:04.036293+0000", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.7120999999999999e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:13.253 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:13.301 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057574781999999998, , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 126.47379893, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057546552000000001, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for subops from 0", , 2021-01-28T21:30:13.351 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:27.384701+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:30.966124+0000", , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 102.286861452, , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.115346143, Ignoring malparsed backtrace: 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPushReply(3.d 47/45 [PushReplyOp(3:bd3332bf:::gibba02740590-173:head)])", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:31.905885+0000", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 101.347100532, Ignoring malparsed backtrace: 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPush(3.9 76/75 [PushOp(3:976daa4c:::gibba02740590-388:head, version: 25'85, data_included: [504309~539013,1796649~509563], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(3:976daa4c:::gibba02740590-388:head@25'85, size: 2504309, copy_subset: [0~2504309], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:2306212, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:29:29.152628+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.449 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.401611+0000", , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.6767999999999997e-05 , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.474 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.699165+0000", , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.29836768899999999 , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:48.844741+0000", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.00096275600000000005 Ignoring malparsed backtrace: 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "started", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:49.500590+0000", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 6.5687999999999999e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise , raise value.with_traceback(tb) , File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run Ignoring malparsed backtrace: 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.6723000000000005e-05 , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.918 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.15017624199999999 , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.983559+0000", , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00013104499999999999 , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", , 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.676623+0000", , 2021-01-28T21:30:13.964 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 Ignoring malparsed backtrace: 2021-01-28T21:30:13.985 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.682198+0000", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.7670000000000003e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.687855+0000", , 2021-01-28T21:30:14.008 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.3838000000000002e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.868039+0000", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.6422000000000003e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.869901+0000", , 2021-01-28T21:30:14.052 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.3876999999999998e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 49.654020881999998, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.191085802, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.097 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:24.415540+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.119 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.141 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.2329999999999997e-06 , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836246+0000", , 2021-01-28T21:30:14.508 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8579999999999999e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836932+0000", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.096930559999999999 Ignoring malparsed backtrace: 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.576 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 2171 Ignoring malparsed backtrace: 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "throttled", , 2021-01-28T21:30:14.600 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.935954+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.621 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.631600+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.635256+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.684 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00053292300000000003 , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.0094789959999999999 , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.768 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.1505000000000003e-05 , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 9665 , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:28.096642+0000", , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8930000000000004e-06 , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:39.086006+0000", , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00098902199999999999 , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:47.343564+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:15.224 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", Ignoring malparsed backtrace: 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.4930000000000001e-06 , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.001762855 , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.338 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.014717160999999999 , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:16.248449+0000", , 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:15.366 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:36.482680+0000", , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00091062800000000002 , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:58.200337+0000", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.53033678399999995 Ignoring malparsed backtrace: 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 41.744552429000002, , 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.017655751000000001, , 2021-01-28T21:30:15.432 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:33.514095+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.087655661999999995, , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:43.952803+0000", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.5390000000000002e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.040258+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.862 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for subops from 4", , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.151551+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.884 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.153686+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "done", , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.158101+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.930 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-01-28T21:30:15.977 INFO:teuthology.orchestra.run.gibba003.stdout: "description": "osd_op(client.4409.0:4502 3.6 3:606549d0:::gibba02740590-302:head [setxattr _header (56) in=7b,truncate 3664091] snapc 0=[] ondisk+write+known_if_redirected e36)", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "initiated_at": "2021-01-28T21:28:16.015999+0000", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 119.758439964, Ignoring malparsed backtrace: 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 1.5741000000000001e-05 , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:16.025 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.73173625200000003 , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.588895+0000", , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.072 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.297023+0000", , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.5664e-05 , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.668766+0000", , 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.37291091900000001 , 2021-01-28T21:30:16.097 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: reweighted osd.7 to 1 (10000) , 2021-01-28T21:30:29.060 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 7 1 , set osd.7 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: instructed osd(s) 2 to deep-scrub , 2021-01-28T21:30:38.281 INFO:tasks.ceph:Scrubbing osd.3 , 2021-01-28T21:30:38.282 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() Ignoring malparsed backtrace: 2021-01-28T21:31:24.207 DEBUG:teuthology.orchestra.run.gibba003:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2021-01-28T21:31:24.228 INFO:tasks.ceph:Compressing logs... , 2021-01-28T21:31:24.229 DEBUG:teuthology.orchestra.run.gibba003:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- Ignoring malparsed backtrace: 2021-01-28T21:31:38.162 DEBUG:teuthology.orchestra.run.gibba027:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . , 2021-01-28T21:32:00.691 INFO:teuthology.task.internal:Transferring binaries for coredumps... , 2021-01-28T21:32:00.720 ERROR:teuthology.run_tasks:Manager failed: internal.archive Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f60cc8c515a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55d61134dfd6] Ignoring malparsed backtrace: 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "op_commit", , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:36.033692+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:12.616 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:12.659 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.702 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.723 INFO:teuthology.orchestra.run.gibba027.stdout: } , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: ] , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:12.744 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0 , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 5.9333000000000003e-05 , 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.788 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "tid": 9733 , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:28.740029+0000", , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 1.3723000000000001e-05 , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.587417+0000", , 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.1121999999999997e-05 , 2021-01-28T21:30:12.852 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "header_read", , 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.121706+0000", , 2021-01-28T21:30:12.873 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.653e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:12.893 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:30:04.036293+0000", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.7120999999999999e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:13.253 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:13.301 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057574781999999998, , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 126.47379893, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057546552000000001, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for subops from 0", , 2021-01-28T21:30:13.351 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:27.384701+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:30.966124+0000", , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 102.286861452, , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.115346143, Ignoring malparsed backtrace: 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPushReply(3.d 47/45 [PushReplyOp(3:bd3332bf:::gibba02740590-173:head)])", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:31.905885+0000", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 101.347100532, Ignoring malparsed backtrace: 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPush(3.9 76/75 [PushOp(3:976daa4c:::gibba02740590-388:head, version: 25'85, data_included: [504309~539013,1796649~509563], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(3:976daa4c:::gibba02740590-388:head@25'85, size: 2504309, copy_subset: [0~2504309], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:2306212, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:29:29.152628+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.449 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.401611+0000", , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.6767999999999997e-05 , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.474 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.699165+0000", , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.29836768899999999 , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:48.844741+0000", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.00096275600000000005 Ignoring malparsed backtrace: 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "started", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:49.500590+0000", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 6.5687999999999999e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise , raise value.with_traceback(tb) , File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run Ignoring malparsed backtrace: 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.6723000000000005e-05 , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.918 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.15017624199999999 , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.983559+0000", , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00013104499999999999 , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", , 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.676623+0000", , 2021-01-28T21:30:13.964 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 Ignoring malparsed backtrace: 2021-01-28T21:30:13.985 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.682198+0000", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.7670000000000003e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.687855+0000", , 2021-01-28T21:30:14.008 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.3838000000000002e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.868039+0000", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.6422000000000003e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.869901+0000", , 2021-01-28T21:30:14.052 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.3876999999999998e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 49.654020881999998, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.191085802, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.097 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:24.415540+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.119 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.141 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.2329999999999997e-06 , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836246+0000", , 2021-01-28T21:30:14.508 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8579999999999999e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836932+0000", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.096930559999999999 Ignoring malparsed backtrace: 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.576 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 2171 Ignoring malparsed backtrace: 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "throttled", , 2021-01-28T21:30:14.600 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.935954+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.621 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.631600+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.635256+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.684 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00053292300000000003 , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.0094789959999999999 , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.768 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.1505000000000003e-05 , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 9665 , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:28.096642+0000", , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8930000000000004e-06 , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:39.086006+0000", , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00098902199999999999 , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:47.343564+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:15.224 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", Ignoring malparsed backtrace: 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.4930000000000001e-06 , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.001762855 , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.338 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.014717160999999999 , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:16.248449+0000", , 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:15.366 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:36.482680+0000", , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00091062800000000002 , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:58.200337+0000", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.53033678399999995 Ignoring malparsed backtrace: 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 41.744552429000002, , 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.017655751000000001, , 2021-01-28T21:30:15.432 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:33.514095+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.087655661999999995, , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:43.952803+0000", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.5390000000000002e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.040258+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.862 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for subops from 4", , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.151551+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.884 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.153686+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "done", , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.158101+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.930 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-01-28T21:30:15.977 INFO:teuthology.orchestra.run.gibba003.stdout: "description": "osd_op(client.4409.0:4502 3.6 3:606549d0:::gibba02740590-302:head [setxattr _header (56) in=7b,truncate 3664091] snapc 0=[] ondisk+write+known_if_redirected e36)", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "initiated_at": "2021-01-28T21:28:16.015999+0000", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 119.758439964, Ignoring malparsed backtrace: 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 1.5741000000000001e-05 , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:16.025 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.73173625200000003 , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.588895+0000", , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.072 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.297023+0000", , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.5664e-05 , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.668766+0000", , 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.37291091900000001 , 2021-01-28T21:30:16.097 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: reweighted osd.7 to 1 (10000) , 2021-01-28T21:30:29.060 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 7 1 , set osd.7 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: instructed osd(s) 2 to deep-scrub , 2021-01-28T21:30:38.281 INFO:tasks.ceph:Scrubbing osd.3 , 2021-01-28T21:30:38.282 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() Ignoring malparsed backtrace: 2021-01-28T21:31:24.207 DEBUG:teuthology.orchestra.run.gibba003:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2021-01-28T21:31:24.228 INFO:tasks.ceph:Compressing logs... , 2021-01-28T21:31:24.229 DEBUG:teuthology.orchestra.run.gibba003:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- Ignoring malparsed backtrace: 2021-01-28T21:31:38.162 DEBUG:teuthology.orchestra.run.gibba027:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . , 2021-01-28T21:32:00.691 INFO:teuthology.task.internal:Transferring binaries for coredumps... , 2021-01-28T21:32:00.720 ERROR:teuthology.run_tasks:Manager failed: internal.archive Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f60cc8c515a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55d61134dfd6] Ignoring malparsed backtrace: 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "op_commit", , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:36.033692+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:12.616 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:12.659 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.702 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.723 INFO:teuthology.orchestra.run.gibba027.stdout: } , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: ] , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:12.744 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0 , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 5.9333000000000003e-05 , 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.788 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "tid": 9733 , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:28.740029+0000", , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 1.3723000000000001e-05 , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.587417+0000", , 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.1121999999999997e-05 , 2021-01-28T21:30:12.852 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "header_read", , 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.121706+0000", , 2021-01-28T21:30:12.873 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.653e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:12.893 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:30:04.036293+0000", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.7120999999999999e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:13.253 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:13.301 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057574781999999998, , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 126.47379893, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057546552000000001, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for subops from 0", , 2021-01-28T21:30:13.351 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:27.384701+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:30.966124+0000", , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 102.286861452, , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.115346143, Ignoring malparsed backtrace: 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPushReply(3.d 47/45 [PushReplyOp(3:bd3332bf:::gibba02740590-173:head)])", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:31.905885+0000", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 101.347100532, Ignoring malparsed backtrace: 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPush(3.9 76/75 [PushOp(3:976daa4c:::gibba02740590-388:head, version: 25'85, data_included: [504309~539013,1796649~509563], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(3:976daa4c:::gibba02740590-388:head@25'85, size: 2504309, copy_subset: [0~2504309], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:2306212, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:29:29.152628+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.449 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.401611+0000", , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.6767999999999997e-05 , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.474 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.699165+0000", , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.29836768899999999 , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:48.844741+0000", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.00096275600000000005 Ignoring malparsed backtrace: 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "started", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:49.500590+0000", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 6.5687999999999999e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise , raise value.with_traceback(tb) , File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run Ignoring malparsed backtrace: 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.6723000000000005e-05 , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.918 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.15017624199999999 , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.983559+0000", , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00013104499999999999 , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", , 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.676623+0000", , 2021-01-28T21:30:13.964 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 Ignoring malparsed backtrace: 2021-01-28T21:30:13.985 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.682198+0000", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.7670000000000003e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.687855+0000", , 2021-01-28T21:30:14.008 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.3838000000000002e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.868039+0000", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.6422000000000003e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.869901+0000", , 2021-01-28T21:30:14.052 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.3876999999999998e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 49.654020881999998, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.191085802, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.097 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:24.415540+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.119 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.141 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.2329999999999997e-06 , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836246+0000", , 2021-01-28T21:30:14.508 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8579999999999999e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836932+0000", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.096930559999999999 Ignoring malparsed backtrace: 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.576 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 2171 Ignoring malparsed backtrace: 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "throttled", , 2021-01-28T21:30:14.600 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.935954+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.621 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.631600+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.635256+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.684 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00053292300000000003 , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.0094789959999999999 , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.768 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.1505000000000003e-05 , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 9665 , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:28.096642+0000", , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8930000000000004e-06 , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:39.086006+0000", , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00098902199999999999 , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:47.343564+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:15.224 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", Ignoring malparsed backtrace: 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.4930000000000001e-06 , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.001762855 , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.338 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.014717160999999999 , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:16.248449+0000", , 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:15.366 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:36.482680+0000", , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00091062800000000002 , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:58.200337+0000", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.53033678399999995 Ignoring malparsed backtrace: 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 41.744552429000002, , 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.017655751000000001, , 2021-01-28T21:30:15.432 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:33.514095+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.087655661999999995, , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:43.952803+0000", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.5390000000000002e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.040258+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.862 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for subops from 4", , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.151551+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.884 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.153686+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "done", , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.158101+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.930 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-01-28T21:30:15.977 INFO:teuthology.orchestra.run.gibba003.stdout: "description": "osd_op(client.4409.0:4502 3.6 3:606549d0:::gibba02740590-302:head [setxattr _header (56) in=7b,truncate 3664091] snapc 0=[] ondisk+write+known_if_redirected e36)", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "initiated_at": "2021-01-28T21:28:16.015999+0000", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 119.758439964, Ignoring malparsed backtrace: 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 1.5741000000000001e-05 , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:16.025 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.73173625200000003 , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.588895+0000", , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.072 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.297023+0000", , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.5664e-05 , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.668766+0000", , 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.37291091900000001 , 2021-01-28T21:30:16.097 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: reweighted osd.7 to 1 (10000) , 2021-01-28T21:30:29.060 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 7 1 , set osd.7 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: instructed osd(s) 2 to deep-scrub , 2021-01-28T21:30:38.281 INFO:tasks.ceph:Scrubbing osd.3 , 2021-01-28T21:30:38.282 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() Ignoring malparsed backtrace: 2021-01-28T21:31:24.207 DEBUG:teuthology.orchestra.run.gibba003:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2021-01-28T21:31:24.228 INFO:tasks.ceph:Compressing logs... , 2021-01-28T21:31:24.229 DEBUG:teuthology.orchestra.run.gibba003:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- Ignoring malparsed backtrace: 2021-01-28T21:31:38.162 DEBUG:teuthology.orchestra.run.gibba027:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . , 2021-01-28T21:32:00.691 INFO:teuthology.task.internal:Transferring binaries for coredumps... , 2021-01-28T21:32:00.720 ERROR:teuthology.run_tasks:Manager failed: internal.archive Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f60cc8c515a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55d61134dfd6] Ignoring malparsed backtrace: 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "op_commit", , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:36.033692+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:12.616 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:12.659 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.702 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.723 INFO:teuthology.orchestra.run.gibba027.stdout: } , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: ] , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:12.744 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0 , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 5.9333000000000003e-05 , 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.788 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "tid": 9733 , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:28.740029+0000", , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 1.3723000000000001e-05 , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.587417+0000", , 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.1121999999999997e-05 , 2021-01-28T21:30:12.852 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "header_read", , 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.121706+0000", , 2021-01-28T21:30:12.873 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.653e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:12.893 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:30:04.036293+0000", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.7120999999999999e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:13.253 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:13.301 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057574781999999998, , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 126.47379893, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057546552000000001, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for subops from 0", , 2021-01-28T21:30:13.351 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:27.384701+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:30.966124+0000", , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 102.286861452, , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.115346143, Ignoring malparsed backtrace: 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPushReply(3.d 47/45 [PushReplyOp(3:bd3332bf:::gibba02740590-173:head)])", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:31.905885+0000", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 101.347100532, Ignoring malparsed backtrace: 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPush(3.9 76/75 [PushOp(3:976daa4c:::gibba02740590-388:head, version: 25'85, data_included: [504309~539013,1796649~509563], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(3:976daa4c:::gibba02740590-388:head@25'85, size: 2504309, copy_subset: [0~2504309], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:2306212, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:29:29.152628+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.449 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.401611+0000", , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.6767999999999997e-05 , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.474 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.699165+0000", , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.29836768899999999 , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:48.844741+0000", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.00096275600000000005 Ignoring malparsed backtrace: 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "started", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:49.500590+0000", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 6.5687999999999999e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise , raise value.with_traceback(tb) , File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run Ignoring malparsed backtrace: 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.6723000000000005e-05 , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.918 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.15017624199999999 , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.983559+0000", , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00013104499999999999 , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", , 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.676623+0000", , 2021-01-28T21:30:13.964 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 Ignoring malparsed backtrace: 2021-01-28T21:30:13.985 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.682198+0000", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.7670000000000003e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.687855+0000", , 2021-01-28T21:30:14.008 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.3838000000000002e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.868039+0000", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.6422000000000003e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.869901+0000", , 2021-01-28T21:30:14.052 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.3876999999999998e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 49.654020881999998, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.191085802, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.097 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:24.415540+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.119 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.141 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.2329999999999997e-06 , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836246+0000", , 2021-01-28T21:30:14.508 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8579999999999999e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836932+0000", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.096930559999999999 Ignoring malparsed backtrace: 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.576 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 2171 Ignoring malparsed backtrace: 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "throttled", , 2021-01-28T21:30:14.600 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.935954+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.621 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.631600+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.635256+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.684 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00053292300000000003 , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.0094789959999999999 , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.768 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.1505000000000003e-05 , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 9665 , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:28.096642+0000", , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8930000000000004e-06 , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:39.086006+0000", , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00098902199999999999 , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:47.343564+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:15.224 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", Ignoring malparsed backtrace: 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.4930000000000001e-06 , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.001762855 , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.338 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.014717160999999999 , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:16.248449+0000", , 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:15.366 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:36.482680+0000", , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00091062800000000002 , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:58.200337+0000", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.53033678399999995 Ignoring malparsed backtrace: 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 41.744552429000002, , 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.017655751000000001, , 2021-01-28T21:30:15.432 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:33.514095+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.087655661999999995, , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:43.952803+0000", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.5390000000000002e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.040258+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.862 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for subops from 4", , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.151551+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.884 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.153686+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "done", , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.158101+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.930 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-01-28T21:30:15.977 INFO:teuthology.orchestra.run.gibba003.stdout: "description": "osd_op(client.4409.0:4502 3.6 3:606549d0:::gibba02740590-302:head [setxattr _header (56) in=7b,truncate 3664091] snapc 0=[] ondisk+write+known_if_redirected e36)", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "initiated_at": "2021-01-28T21:28:16.015999+0000", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 119.758439964, Ignoring malparsed backtrace: 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 1.5741000000000001e-05 , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:16.025 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.73173625200000003 , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.588895+0000", , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.072 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.297023+0000", , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.5664e-05 , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.668766+0000", , 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.37291091900000001 , 2021-01-28T21:30:16.097 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: reweighted osd.7 to 1 (10000) , 2021-01-28T21:30:29.060 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 7 1 , set osd.7 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: instructed osd(s) 2 to deep-scrub , 2021-01-28T21:30:38.281 INFO:tasks.ceph:Scrubbing osd.3 , 2021-01-28T21:30:38.282 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() Ignoring malparsed backtrace: 2021-01-28T21:31:24.207 DEBUG:teuthology.orchestra.run.gibba003:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2021-01-28T21:31:24.228 INFO:tasks.ceph:Compressing logs... , 2021-01-28T21:31:24.229 DEBUG:teuthology.orchestra.run.gibba003:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- Ignoring malparsed backtrace: 2021-01-28T21:31:38.162 DEBUG:teuthology.orchestra.run.gibba027:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . , 2021-01-28T21:32:00.691 INFO:teuthology.task.internal:Transferring binaries for coredumps... , 2021-01-28T21:32:00.720 ERROR:teuthology.run_tasks:Manager failed: internal.archive Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f60cc8c515a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55d61134dfd6] Ignoring malparsed backtrace: 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "op_commit", , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:36.033692+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:12.616 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:12.659 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.702 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.723 INFO:teuthology.orchestra.run.gibba027.stdout: } , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: ] , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:12.744 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0 , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 5.9333000000000003e-05 , 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.788 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "tid": 9733 , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:28.740029+0000", , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 1.3723000000000001e-05 , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.587417+0000", , 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.1121999999999997e-05 , 2021-01-28T21:30:12.852 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "header_read", , 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.121706+0000", , 2021-01-28T21:30:12.873 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.653e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:12.893 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:30:04.036293+0000", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.7120999999999999e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:13.253 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:13.301 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057574781999999998, , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 126.47379893, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057546552000000001, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for subops from 0", , 2021-01-28T21:30:13.351 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:27.384701+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:30.966124+0000", , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 102.286861452, , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.115346143, Ignoring malparsed backtrace: 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPushReply(3.d 47/45 [PushReplyOp(3:bd3332bf:::gibba02740590-173:head)])", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:31.905885+0000", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 101.347100532, Ignoring malparsed backtrace: 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPush(3.9 76/75 [PushOp(3:976daa4c:::gibba02740590-388:head, version: 25'85, data_included: [504309~539013,1796649~509563], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(3:976daa4c:::gibba02740590-388:head@25'85, size: 2504309, copy_subset: [0~2504309], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:2306212, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:29:29.152628+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.449 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.401611+0000", , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.6767999999999997e-05 , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.474 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.699165+0000", , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.29836768899999999 , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:48.844741+0000", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.00096275600000000005 Ignoring malparsed backtrace: 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "started", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:49.500590+0000", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 6.5687999999999999e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise , raise value.with_traceback(tb) , File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run Ignoring malparsed backtrace: 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.6723000000000005e-05 , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.918 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.15017624199999999 , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.983559+0000", , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00013104499999999999 , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", , 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.676623+0000", , 2021-01-28T21:30:13.964 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 Ignoring malparsed backtrace: 2021-01-28T21:30:13.985 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.682198+0000", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.7670000000000003e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.687855+0000", , 2021-01-28T21:30:14.008 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.3838000000000002e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.868039+0000", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.6422000000000003e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.869901+0000", , 2021-01-28T21:30:14.052 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.3876999999999998e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 49.654020881999998, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.191085802, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.097 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:24.415540+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.119 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.141 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.2329999999999997e-06 , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836246+0000", , 2021-01-28T21:30:14.508 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8579999999999999e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836932+0000", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.096930559999999999 Ignoring malparsed backtrace: 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.576 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 2171 Ignoring malparsed backtrace: 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "throttled", , 2021-01-28T21:30:14.600 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.935954+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.621 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.631600+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.635256+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.684 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00053292300000000003 , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.0094789959999999999 , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.768 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.1505000000000003e-05 , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 9665 , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:28.096642+0000", , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8930000000000004e-06 , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:39.086006+0000", , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00098902199999999999 , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:47.343564+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:15.224 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", Ignoring malparsed backtrace: 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.4930000000000001e-06 , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.001762855 , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.338 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.014717160999999999 , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:16.248449+0000", , 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:15.366 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:36.482680+0000", , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00091062800000000002 , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:58.200337+0000", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.53033678399999995 Ignoring malparsed backtrace: 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 41.744552429000002, , 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.017655751000000001, , 2021-01-28T21:30:15.432 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:33.514095+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.087655661999999995, , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:43.952803+0000", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.5390000000000002e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.040258+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.862 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for subops from 4", , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.151551+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.884 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.153686+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "done", , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.158101+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.930 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-01-28T21:30:15.977 INFO:teuthology.orchestra.run.gibba003.stdout: "description": "osd_op(client.4409.0:4502 3.6 3:606549d0:::gibba02740590-302:head [setxattr _header (56) in=7b,truncate 3664091] snapc 0=[] ondisk+write+known_if_redirected e36)", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "initiated_at": "2021-01-28T21:28:16.015999+0000", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 119.758439964, Ignoring malparsed backtrace: 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 1.5741000000000001e-05 , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:16.025 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.73173625200000003 , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.588895+0000", , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.072 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.297023+0000", , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.5664e-05 , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.668766+0000", , 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.37291091900000001 , 2021-01-28T21:30:16.097 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: reweighted osd.7 to 1 (10000) , 2021-01-28T21:30:29.060 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 7 1 , set osd.7 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: instructed osd(s) 2 to deep-scrub , 2021-01-28T21:30:38.281 INFO:tasks.ceph:Scrubbing osd.3 , 2021-01-28T21:30:38.282 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() Ignoring malparsed backtrace: 2021-01-28T21:31:24.207 DEBUG:teuthology.orchestra.run.gibba003:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2021-01-28T21:31:24.228 INFO:tasks.ceph:Compressing logs... , 2021-01-28T21:31:24.229 DEBUG:teuthology.orchestra.run.gibba003:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- Ignoring malparsed backtrace: 2021-01-28T21:31:38.162 DEBUG:teuthology.orchestra.run.gibba027:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . , 2021-01-28T21:32:00.691 INFO:teuthology.task.internal:Transferring binaries for coredumps... , 2021-01-28T21:32:00.720 ERROR:teuthology.run_tasks:Manager failed: internal.archive Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f60cc8c515a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55d61134dfd6] Ignoring malparsed backtrace: 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "op_commit", , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:36.033692+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:12.616 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:12.659 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.702 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.723 INFO:teuthology.orchestra.run.gibba027.stdout: } , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: ] , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:12.744 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0 , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 5.9333000000000003e-05 , 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.788 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "tid": 9733 , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:28.740029+0000", , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 1.3723000000000001e-05 , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.587417+0000", , 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.1121999999999997e-05 , 2021-01-28T21:30:12.852 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "header_read", , 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.121706+0000", , 2021-01-28T21:30:12.873 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.653e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:12.893 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:30:04.036293+0000", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.7120999999999999e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:13.253 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:13.301 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057574781999999998, , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 126.47379893, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057546552000000001, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for subops from 0", , 2021-01-28T21:30:13.351 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:27.384701+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:30.966124+0000", , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 102.286861452, , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.115346143, Ignoring malparsed backtrace: 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPushReply(3.d 47/45 [PushReplyOp(3:bd3332bf:::gibba02740590-173:head)])", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:31.905885+0000", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 101.347100532, Ignoring malparsed backtrace: 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPush(3.9 76/75 [PushOp(3:976daa4c:::gibba02740590-388:head, version: 25'85, data_included: [504309~539013,1796649~509563], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(3:976daa4c:::gibba02740590-388:head@25'85, size: 2504309, copy_subset: [0~2504309], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:2306212, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:29:29.152628+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.449 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.401611+0000", , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.6767999999999997e-05 , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.474 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.699165+0000", , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.29836768899999999 , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:48.844741+0000", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.00096275600000000005 Ignoring malparsed backtrace: 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "started", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:49.500590+0000", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 6.5687999999999999e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise , raise value.with_traceback(tb) , File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run Ignoring malparsed backtrace: 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.6723000000000005e-05 , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.918 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.15017624199999999 , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.983559+0000", , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00013104499999999999 , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", , 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.676623+0000", , 2021-01-28T21:30:13.964 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 Ignoring malparsed backtrace: 2021-01-28T21:30:13.985 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.682198+0000", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.7670000000000003e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.687855+0000", , 2021-01-28T21:30:14.008 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.3838000000000002e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.868039+0000", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.6422000000000003e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.869901+0000", , 2021-01-28T21:30:14.052 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.3876999999999998e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 49.654020881999998, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.191085802, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.097 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:24.415540+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.119 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.141 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.2329999999999997e-06 , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836246+0000", , 2021-01-28T21:30:14.508 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8579999999999999e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836932+0000", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.096930559999999999 Ignoring malparsed backtrace: 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.576 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 2171 Ignoring malparsed backtrace: 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "throttled", , 2021-01-28T21:30:14.600 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.935954+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.621 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.631600+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.635256+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.684 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00053292300000000003 , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.0094789959999999999 , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.768 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.1505000000000003e-05 , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 9665 , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:28.096642+0000", , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8930000000000004e-06 , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:39.086006+0000", , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00098902199999999999 , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:47.343564+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:15.224 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", Ignoring malparsed backtrace: 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.4930000000000001e-06 , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.001762855 , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.338 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.014717160999999999 , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:16.248449+0000", , 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:15.366 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:36.482680+0000", , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00091062800000000002 , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:58.200337+0000", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.53033678399999995 Ignoring malparsed backtrace: 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 41.744552429000002, , 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.017655751000000001, , 2021-01-28T21:30:15.432 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:33.514095+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.087655661999999995, , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:43.952803+0000", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.5390000000000002e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.040258+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.862 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for subops from 4", , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.151551+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.884 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.153686+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "done", , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.158101+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.930 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-01-28T21:30:15.977 INFO:teuthology.orchestra.run.gibba003.stdout: "description": "osd_op(client.4409.0:4502 3.6 3:606549d0:::gibba02740590-302:head [setxattr _header (56) in=7b,truncate 3664091] snapc 0=[] ondisk+write+known_if_redirected e36)", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "initiated_at": "2021-01-28T21:28:16.015999+0000", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 119.758439964, Ignoring malparsed backtrace: 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 1.5741000000000001e-05 , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:16.025 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.73173625200000003 , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.588895+0000", , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.072 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.297023+0000", , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.5664e-05 , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.668766+0000", , 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.37291091900000001 , 2021-01-28T21:30:16.097 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: reweighted osd.7 to 1 (10000) , 2021-01-28T21:30:29.060 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 7 1 , set osd.7 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: instructed osd(s) 2 to deep-scrub , 2021-01-28T21:30:38.281 INFO:tasks.ceph:Scrubbing osd.3 , 2021-01-28T21:30:38.282 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() Ignoring malparsed backtrace: 2021-01-28T21:31:24.207 DEBUG:teuthology.orchestra.run.gibba003:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2021-01-28T21:31:24.228 INFO:tasks.ceph:Compressing logs... , 2021-01-28T21:31:24.229 DEBUG:teuthology.orchestra.run.gibba003:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- Ignoring malparsed backtrace: 2021-01-28T21:31:38.162 DEBUG:teuthology.orchestra.run.gibba027:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . , 2021-01-28T21:32:00.691 INFO:teuthology.task.internal:Transferring binaries for coredumps... , 2021-01-28T21:32:00.720 ERROR:teuthology.run_tasks:Manager failed: internal.archive Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f60cc8c515a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55d61134dfd6] Ignoring malparsed backtrace: 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "op_commit", , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:36.033692+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:12.616 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:12.659 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.702 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.723 INFO:teuthology.orchestra.run.gibba027.stdout: } , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: ] , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:12.744 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0 , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 5.9333000000000003e-05 , 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.788 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "tid": 9733 , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:28.740029+0000", , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 1.3723000000000001e-05 , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.587417+0000", , 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.1121999999999997e-05 , 2021-01-28T21:30:12.852 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "header_read", , 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.121706+0000", , 2021-01-28T21:30:12.873 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.653e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:12.893 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:30:04.036293+0000", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.7120999999999999e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:13.253 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:13.301 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057574781999999998, , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 126.47379893, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057546552000000001, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for subops from 0", , 2021-01-28T21:30:13.351 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:27.384701+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:30.966124+0000", , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 102.286861452, , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.115346143, Ignoring malparsed backtrace: 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPushReply(3.d 47/45 [PushReplyOp(3:bd3332bf:::gibba02740590-173:head)])", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:31.905885+0000", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 101.347100532, Ignoring malparsed backtrace: 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPush(3.9 76/75 [PushOp(3:976daa4c:::gibba02740590-388:head, version: 25'85, data_included: [504309~539013,1796649~509563], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(3:976daa4c:::gibba02740590-388:head@25'85, size: 2504309, copy_subset: [0~2504309], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:2306212, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:29:29.152628+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.449 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.401611+0000", , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.6767999999999997e-05 , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.474 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.699165+0000", , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.29836768899999999 , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:48.844741+0000", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.00096275600000000005 Ignoring malparsed backtrace: 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "started", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:49.500590+0000", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 6.5687999999999999e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise , raise value.with_traceback(tb) , File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run Ignoring malparsed backtrace: 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.6723000000000005e-05 , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.918 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.15017624199999999 , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.983559+0000", , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00013104499999999999 , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", , 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.676623+0000", , 2021-01-28T21:30:13.964 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 Ignoring malparsed backtrace: 2021-01-28T21:30:13.985 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.682198+0000", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.7670000000000003e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.687855+0000", , 2021-01-28T21:30:14.008 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.3838000000000002e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.868039+0000", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.6422000000000003e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.869901+0000", , 2021-01-28T21:30:14.052 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.3876999999999998e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 49.654020881999998, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.191085802, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.097 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:24.415540+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.119 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.141 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.2329999999999997e-06 , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836246+0000", , 2021-01-28T21:30:14.508 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8579999999999999e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836932+0000", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.096930559999999999 Ignoring malparsed backtrace: 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.576 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 2171 Ignoring malparsed backtrace: 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "throttled", , 2021-01-28T21:30:14.600 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.935954+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.621 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.631600+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.635256+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.684 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00053292300000000003 , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.0094789959999999999 , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.768 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.1505000000000003e-05 , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 9665 , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:28.096642+0000", , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8930000000000004e-06 , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:39.086006+0000", , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00098902199999999999 , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:47.343564+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:15.224 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", Ignoring malparsed backtrace: 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.4930000000000001e-06 , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.001762855 , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.338 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.014717160999999999 , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:16.248449+0000", , 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:15.366 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:36.482680+0000", , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00091062800000000002 , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:58.200337+0000", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.53033678399999995 Ignoring malparsed backtrace: 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 41.744552429000002, , 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.017655751000000001, , 2021-01-28T21:30:15.432 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:33.514095+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.087655661999999995, , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:43.952803+0000", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.5390000000000002e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.040258+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.862 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for subops from 4", , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.151551+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.884 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.153686+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "done", , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.158101+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.930 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-01-28T21:30:15.977 INFO:teuthology.orchestra.run.gibba003.stdout: "description": "osd_op(client.4409.0:4502 3.6 3:606549d0:::gibba02740590-302:head [setxattr _header (56) in=7b,truncate 3664091] snapc 0=[] ondisk+write+known_if_redirected e36)", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "initiated_at": "2021-01-28T21:28:16.015999+0000", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 119.758439964, Ignoring malparsed backtrace: 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 1.5741000000000001e-05 , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:16.025 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.73173625200000003 , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.588895+0000", , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.072 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.297023+0000", , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.5664e-05 , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.668766+0000", , 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.37291091900000001 , 2021-01-28T21:30:16.097 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: reweighted osd.7 to 1 (10000) , 2021-01-28T21:30:29.060 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 7 1 , set osd.7 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: instructed osd(s) 2 to deep-scrub , 2021-01-28T21:30:38.281 INFO:tasks.ceph:Scrubbing osd.3 , 2021-01-28T21:30:38.282 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() Ignoring malparsed backtrace: 2021-01-28T21:31:24.207 DEBUG:teuthology.orchestra.run.gibba003:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2021-01-28T21:31:24.228 INFO:tasks.ceph:Compressing logs... , 2021-01-28T21:31:24.229 DEBUG:teuthology.orchestra.run.gibba003:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- Ignoring malparsed backtrace: 2021-01-28T21:31:38.162 DEBUG:teuthology.orchestra.run.gibba027:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . , 2021-01-28T21:32:00.691 INFO:teuthology.task.internal:Transferring binaries for coredumps... , 2021-01-28T21:32:00.720 ERROR:teuthology.run_tasks:Manager failed: internal.archive Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f60cc8c515a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55d61134dfd6] Ignoring malparsed backtrace: 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "op_commit", , 2021-01-28T21:30:12.595 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:36.033692+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:12.616 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", , 2021-01-28T21:30:12.617 INFO:teuthology.orchestra.run.gibba027.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.638 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:12.659 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.660 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.681 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.702 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:12.703 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:12.723 INFO:teuthology.orchestra.run.gibba027.stdout: } , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: ] , 2021-01-28T21:30:12.724 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:12.744 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client_info": { , 2021-01-28T21:30:12.745 INFO:teuthology.orchestra.run.gibba027.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0 , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.766 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 5.9333000000000003e-05 , 2021-01-28T21:30:12.787 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.788 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "tid": 9733 , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: }, , 2021-01-28T21:30:12.809 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:28.740029+0000", , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 1.3723000000000001e-05 , 2021-01-28T21:30:12.830 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.587417+0000", , 2021-01-28T21:30:12.851 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.1121999999999997e-05 , 2021-01-28T21:30:12.852 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "header_read", , 2021-01-28T21:30:12.872 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:39.121706+0000", , 2021-01-28T21:30:12.873 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.653e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:12.893 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:30:04.036293+0000", , 2021-01-28T21:30:12.894 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 3.7120999999999999e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:13.253 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ , 2021-01-28T21:30:13.254 INFO:teuthology.orchestra.run.gibba027.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", , 2021-01-28T21:30:13.277 INFO:teuthology.orchestra.run.gibba027.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:13.301 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057574781999999998, , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { , 2021-01-28T21:30:13.302 INFO:teuthology.orchestra.run.gibba027.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 126.47379893, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.057546552000000001, , 2021-01-28T21:30:13.326 INFO:teuthology.orchestra.run.gibba027.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.350 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "waiting for subops from 0", , 2021-01-28T21:30:13.351 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:28:27.384701+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:30.966124+0000", , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 102.286861452, , 2021-01-28T21:30:13.376 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.115346143, Ignoring malparsed backtrace: 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPushReply(3.d 47/45 [PushReplyOp(3:bd3332bf:::gibba02740590-173:head)])", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:28:31.905885+0000", , 2021-01-28T21:30:13.401 INFO:teuthology.orchestra.run.gibba027.stdout: "age": 101.347100532, Ignoring malparsed backtrace: 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: { , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "description": "MOSDPGPush(3.9 76/75 [PushOp(3:976daa4c:::gibba02740590-388:head, version: 25'85, data_included: [504309~539013,1796649~509563], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 3, recovery_info: ObjectRecoveryInfo(3:976daa4c:::gibba02740590-388:head@25'85, size: 2504309, copy_subset: [0~2504309], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:2306212, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", , 2021-01-28T21:30:13.425 INFO:teuthology.orchestra.run.gibba027.stdout: "initiated_at": "2021-01-28T21:29:29.152628+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:13.449 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.401611+0000", , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 4.6767999999999997e-05 , 2021-01-28T21:30:13.450 INFO:teuthology.orchestra.run.gibba027.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.474 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:40.699165+0000", , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.29836768899999999 , 2021-01-28T21:30:13.475 INFO:teuthology.orchestra.run.gibba027.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:48.844741+0000", , 2021-01-28T21:30:13.498 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 0.00096275600000000005 Ignoring malparsed backtrace: 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "event": "started", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "time": "2021-01-28T21:29:49.500590+0000", , 2021-01-28T21:30:13.521 INFO:teuthology.orchestra.run.gibba027.stdout: "duration": 6.5687999999999999e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise , raise value.with_traceback(tb) , File "src/gevent/greenlet.py", line 766, in gevent._greenlet.Greenlet.run Ignoring malparsed backtrace: 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.6723000000000005e-05 , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:13.896 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:13.918 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.15017624199999999 , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:13.919 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.983559+0000", , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00013104499999999999 , 2021-01-28T21:30:13.941 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", , 2021-01-28T21:30:13.963 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.676623+0000", , 2021-01-28T21:30:13.964 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 Ignoring malparsed backtrace: 2021-01-28T21:30:13.985 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.682198+0000", , 2021-01-28T21:30:13.986 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.7670000000000003e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.007 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.687855+0000", , 2021-01-28T21:30:14.008 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.3838000000000002e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.868039+0000", , 2021-01-28T21:30:14.029 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.6422000000000003e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "reached_pg", , 2021-01-28T21:30:14.051 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:09.869901+0000", , 2021-01-28T21:30:14.052 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.3876999999999998e-05 Ignoring malparsed backtrace: 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 49.654020881999998, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.191085802, , 2021-01-28T21:30:14.074 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.096 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.097 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:24.415540+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.118 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.119 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.140 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.141 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:14.163 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 5.2329999999999997e-06 , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.185 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:14.507 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836246+0000", , 2021-01-28T21:30:14.508 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8579999999999999e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.836932+0000", , 2021-01-28T21:30:14.532 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.096930559999999999 Ignoring malparsed backtrace: 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.553 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.576 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", , 2021-01-28T21:30:14.577 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 2171 Ignoring malparsed backtrace: 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.599 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "throttled", , 2021-01-28T21:30:14.600 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:48.935954+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.620 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "queued_for_pg", , 2021-01-28T21:30:14.621 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.631600+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:14.641 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:32.635256+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", , 2021-01-28T21:30:14.662 INFO:teuthology.orchestra.run.gibba003.stdout: "client_addr": "172.21.2.127:0/1198896021", Ignoring malparsed backtrace: 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.683 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.684 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:14.704 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00053292300000000003 , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.726 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.0094789959999999999 , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.748 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.768 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.1505000000000003e-05 , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.769 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "tid": 9665 , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:14.790 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ Ignoring malparsed backtrace: 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:28.096642+0000", , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 4.8930000000000004e-06 , 2021-01-28T21:30:14.811 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:39.086006+0000", , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00098902199999999999 , 2021-01-28T21:30:14.832 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "op_commit", , 2021-01-28T21:30:15.159 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:47.343564+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: ] , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.181 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.203 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", Ignoring malparsed backtrace: 2021-01-28T21:30:15.224 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.225 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", Ignoring malparsed backtrace: 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.246 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client_info": { , 2021-01-28T21:30:15.268 INFO:teuthology.orchestra.run.gibba003.stdout: "client": "client.4409", Ignoring malparsed backtrace: 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.4930000000000001e-06 , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.295 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.001762855 , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.317 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.338 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.014717160999999999 , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.339 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:16.248449+0000", , 2021-01-28T21:30:15.365 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:15.366 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:36.482680+0000", , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.00091062800000000002 , 2021-01-28T21:30:15.388 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "all_read", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:58.200337+0000", , 2021-01-28T21:30:15.410 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.53033678399999995 Ignoring malparsed backtrace: 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 41.744552429000002, , 2021-01-28T21:30:15.431 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.017655751000000001, , 2021-01-28T21:30:15.432 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { Ignoring malparsed backtrace: 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "started", , 2021-01-28T21:30:15.454 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:33.514095+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.087655661999999995, , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "type_data": { , 2021-01-28T21:30:15.775 INFO:teuthology.orchestra.run.gibba003.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:43.952803+0000", , 2021-01-28T21:30:15.797 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 2.5390000000000002e-06 Ignoring malparsed backtrace: 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.819 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "header_read", , 2021-01-28T21:30:15.841 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.040258+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.862 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "waiting for subops from 4", , 2021-01-28T21:30:15.863 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.151551+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.884 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "sub_op_commit_rec", , 2021-01-28T21:30:15.885 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.153686+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "done", , 2021-01-28T21:30:15.907 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:27:44.158101+0000", Ignoring malparsed backtrace: 2021-01-28T21:30:15.930 INFO:teuthology.orchestra.run.gibba003.stdout: "events": [ , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.931 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: { , 2021-01-28T21:30:15.954 INFO:teuthology.orchestra.run.gibba003.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-01-28T21:30:15.977 INFO:teuthology.orchestra.run.gibba003.stdout: "description": "osd_op(client.4409.0:4502 3.6 3:606549d0:::gibba02740590-302:head [setxattr _header (56) in=7b,truncate 3664091] snapc 0=[] ondisk+write+known_if_redirected e36)", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "initiated_at": "2021-01-28T21:28:16.015999+0000", , 2021-01-28T21:30:15.978 INFO:teuthology.orchestra.run.gibba003.stdout: "age": 119.758439964, Ignoring malparsed backtrace: 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 1.5741000000000001e-05 , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: }, , 2021-01-28T21:30:16.002 INFO:teuthology.orchestra.run.gibba003.stdout: { Ignoring malparsed backtrace: 2021-01-28T21:30:16.025 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.73173625200000003 , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: } , 2021-01-28T21:30:16.026 INFO:teuthology.orchestra.run.gibba003.stdout: ] Ignoring malparsed backtrace: 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:28:50.588895+0000", , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0 , 2021-01-28T21:30:16.049 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.072 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.297023+0000", , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 6.5664e-05 , 2021-01-28T21:30:16.073 INFO:teuthology.orchestra.run.gibba003.stdout: }, Ignoring malparsed backtrace: 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "time": "2021-01-28T21:29:58.668766+0000", , 2021-01-28T21:30:16.096 INFO:teuthology.orchestra.run.gibba003.stdout: "duration": 0.37291091900000001 , 2021-01-28T21:30:16.097 INFO:teuthology.orchestra.run.gibba003.stdout: } Ignoring malparsed backtrace: reweighted osd.7 to 1 (10000) , 2021-01-28T21:30:29.060 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 7 1 , set osd.7 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: instructed osd(s) 2 to deep-scrub , 2021-01-28T21:30:38.281 INFO:tasks.ceph:Scrubbing osd.3 , 2021-01-28T21:30:38.282 DEBUG:teuthology.orchestra.run.gibba027:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() Ignoring malparsed backtrace: 2021-01-28T21:31:24.207 DEBUG:teuthology.orchestra.run.gibba003:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2021-01-28T21:31:24.228 INFO:tasks.ceph:Compressing logs... , 2021-01-28T21:31:24.229 DEBUG:teuthology.orchestra.run.gibba003:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- Ignoring malparsed backtrace: 2021-01-28T21:31:38.162 DEBUG:teuthology.orchestra.run.gibba027:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . , 2021-01-28T21:32:00.691 INFO:teuthology.task.internal:Transferring binaries for coredumps... , 2021-01-28T21:32:00.720 ERROR:teuthology.run_tasks:Manager failed: internal.archive Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f05bc88815a] , 2: (CopyFromOp::_finish(TestOp::CallbackInfo*)+0x4f4) [0x556ba561b904] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-29T11:19:16.605 DEBUG:teuthology.orchestra.run.gibba019:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-29T11:19:16.850 INFO:teuthology.orchestra.run.gibba019.stdout: Ignoring malparsed backtrace: 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~0s Ignoring malparsed backtrace: 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:=============================================================================== , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:^- hv01.front.sepia.ceph.com 3 6 377 23 -3024us[-3028us] +/- 118ms Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f05bc88815a] , 2: (CopyFromOp::_finish(TestOp::CallbackInfo*)+0x4f4) [0x556ba561b904] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-29T11:19:16.605 DEBUG:teuthology.orchestra.run.gibba019:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-29T11:19:16.850 INFO:teuthology.orchestra.run.gibba019.stdout: Ignoring malparsed backtrace: 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~0s Ignoring malparsed backtrace: 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:=============================================================================== , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:^- hv01.front.sepia.ceph.com 3 6 377 23 -3024us[-3028us] +/- 118ms Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f05bc88815a] , 2: (CopyFromOp::_finish(TestOp::CallbackInfo*)+0x4f4) [0x556ba561b904] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-29T11:19:16.605 DEBUG:teuthology.orchestra.run.gibba019:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-29T11:19:16.850 INFO:teuthology.orchestra.run.gibba019.stdout: Ignoring malparsed backtrace: 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~0s Ignoring malparsed backtrace: 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:=============================================================================== , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:^- hv01.front.sepia.ceph.com 3 6 377 23 -3024us[-3028us] +/- 118ms Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f05bc88815a] , 2: (CopyFromOp::_finish(TestOp::CallbackInfo*)+0x4f4) [0x556ba561b904] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-29T11:19:16.605 DEBUG:teuthology.orchestra.run.gibba019:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-29T11:19:16.850 INFO:teuthology.orchestra.run.gibba019.stdout: Ignoring malparsed backtrace: 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~0s Ignoring malparsed backtrace: 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:=============================================================================== , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:^- hv01.front.sepia.ceph.com 3 6 377 23 -3024us[-3028us] +/- 118ms Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f05bc88815a] , 2: (CopyFromOp::_finish(TestOp::CallbackInfo*)+0x4f4) [0x556ba561b904] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-29T11:19:16.605 DEBUG:teuthology.orchestra.run.gibba019:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-29T11:19:16.850 INFO:teuthology.orchestra.run.gibba019.stdout: Ignoring malparsed backtrace: 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~0s Ignoring malparsed backtrace: 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:=============================================================================== , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:^- hv01.front.sepia.ceph.com 3 6 377 23 -3024us[-3028us] +/- 118ms Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f05bc88815a] , 2: (CopyFromOp::_finish(TestOp::CallbackInfo*)+0x4f4) [0x556ba561b904] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-29T11:19:16.605 DEBUG:teuthology.orchestra.run.gibba019:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-29T11:19:16.850 INFO:teuthology.orchestra.run.gibba019.stdout: Ignoring malparsed backtrace: 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~0s Ignoring malparsed backtrace: 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:=============================================================================== , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:^- hv01.front.sepia.ceph.com 3 6 377 23 -3024us[-3028us] +/- 118ms Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f05bc88815a] , 2: (CopyFromOp::_finish(TestOp::CallbackInfo*)+0x4f4) [0x556ba561b904] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-29T11:19:16.605 DEBUG:teuthology.orchestra.run.gibba019:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-29T11:19:16.850 INFO:teuthology.orchestra.run.gibba019.stdout: Ignoring malparsed backtrace: 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~0s Ignoring malparsed backtrace: 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:=============================================================================== , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:^- hv01.front.sepia.ceph.com 3 6 377 23 -3024us[-3028us] +/- 118ms Ignoring malparsed backtrace: ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f05bc88815a] , 2: (CopyFromOp::_finish(TestOp::CallbackInfo*)+0x4f4) [0x556ba561b904] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-29T11:19:16.605 DEBUG:teuthology.orchestra.run.gibba019:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-29T11:19:16.850 INFO:teuthology.orchestra.run.gibba019.stdout: Ignoring malparsed backtrace: 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~0s , 2021-01-29T11:20:18.628 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~0s Ignoring malparsed backtrace: 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:=============================================================================== , 2021-01-29T11:24:05.611 INFO:teuthology.orchestra.run.gibba026.stdout:^- hv01.front.sepia.ceph.com 3 6 377 23 -3024us[-3028us] +/- 118ms Found 11 distinct failure reasons Failure: Unterminated string starting at: line 1 column 278519 (char 278518) 13 jobs: ['5830145', '5829982', '5829821', '5829960', '5829865', '5830147', '5829792', '5830030', '5830075', '5830059', '5829870', '5829979', '5829887'] suites intersection: ['fixed-2'] suites union: ['fixed-2', 'mode/root', 'mon_election/classic', 'mon_election/connectivity', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'rados/cephadm/smoke/{distro/ubuntu_18.04_podman', 'rados/cephadm/smoke/{distro/ubuntu_20.04_podman', 'rados/cephadm/with-work/{distro/ubuntu_18.04_podman', 'rados/cephadm/with-work/{distro/ubuntu_20.04_podman', 'start', 'start}', 'tasks/rados_api_tests}', 'tasks/rados_python}'] Failure: Could not find an image for rhel 8.0 9 jobs: ['5830097', '5830111', '5830081', '5829832', '5829925', '5829853', '5830050', '5829918', '5830020'] Failure: Test failure: test_pwd_expiration_date_update (tasks.mgr.dashboard.test_user.UserTest) 4 jobs: ['5829857', '5830089', '5829922', '5830028'] suites intersection: ['debug/mgr', 'rados/dashboard/{clusters/{2-node-mgr}', 'tasks/dashboard}'] suites union: ['debug/mgr', 'mon_election/classic', 'mon_election/connectivity', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'rados/dashboard/{clusters/{2-node-mgr}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/dashboard}'] Failure: Test failure: test_create_and_delete_export (tasks.cephfs.test_nfs.TestNFS) 2 jobs: ['5830032', '5830151'] suites intersection: ['mon_election/connectivity', 'task/test_orch_cli}'] suites union: ['mon_election/connectivity', 'rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'rados/cephadm/workunits/{distro/ubuntu_20.04_podman', 'task/test_orch_cli}'] Crash: Test failure: test_pool_removal (tasks.mgr.test_progress.TestProgress) ceph version 17.0.0-63-gd3a78fa1 (d3a78fa196cd8226153e77705dbc65cbf19a5200) pacific (dev) 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x12980) [0x7fb832896980] 2: gsignal() 3: abort() 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x1a7) [0x55a3eae1b3df] 5: (KernelDevice::_aio_thread()+0xbb9) [0x55a3eb8fb7b9] 6: (KernelDevice::AioCompletionThread::entry()+0xd) [0x55a3eb8fda7d] 7: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76db) [0x7fb83288b6db] 8: clone() 1 jobs: ['5829938'] suites: ['debug/mgr', 'mon_election/connectivity', 'objectstore/bluestore-stupid', 'rados/mgr/{clusters/{2-node-mgr}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/progress}'] Dead: b'2021-01-29T21:05:14.393 DEBUG:teuthology.exit:Finished running handlers' 8 jobs: ['5830101', '5830120', '5830056', '5829843', '5830011', '5829966', '5829965', '5830029'] suites intersection: [] suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-partial-recovery}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'fast/normal', 'fixed-2', 'mode/packaged', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr/async', 'msgr/async-v2only', 'objectstore/bluestore-basic-min-osd-mem-target', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack', 'openstack}', 'rados', 'rados/cephadm/with-work/{distro/ubuntu_20.04', 'rados/cephadm/workunits/{distro/ubuntu_20.04_podman', 'rados/perf/{ceph', 'rados/thrash-erasure-code/{ceph', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'recovery-overrides/{more-partial-recovery}', 'scheduler/dmclock_1Shard_16Threads', 'scheduler/dmclock_default_shards', 'settings/optimized', 'start', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{ubuntu_latest}', 'task/test_cephadm}', 'tasks/rados_python}', 'thrashers/default', 'thrashers/minsize_recovery', 'thrashosds-health', 'ubuntu_latest', 'workloads/cosbench_64K_read_write}', 'workloads/cosbench_64K_write}', 'workloads/ec-radosbench}', 'workloads/radosbench-high-concurrency}', 'workloads/radosbench_omap_write}'] Failure: "2021-01-28T19:52:12.650801+0000 mon.b (mon.0) 35 : cluster [WRN] Health check failed: 1/3 mons down, quorum b,a (MON_DOWN)" in cluster log 1 jobs: ['5829795'] suites: ['debug/mgr', 'mon_election/connectivity', 'objectstore/bluestore-comp-zlib', 'rados/mgr/{clusters/{2-node-mgr}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/crash}'] Failure: Found coredumps on ubuntu@gibba019.front.sepia.ceph.com 2 jobs: ['5830061', '5829929'] suites intersection: ['mon_election/classic', 'task/test_adoption}'] suites union: ['mon_election/classic', 'rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'rados/cephadm/workunits/{distro/ubuntu_20.04_podman', 'task/test_adoption}'] Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --write-fadvise-dontneed --max-ops 4000 --objects 500 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 50 --op delete 10 --op write_excl 50 --pool unique_pool_0' 2 jobs: ['5829840', '5830119'] suites intersection: ['rados'] suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{more-partial-recovery}', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/crush-compat', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/many', 'msgr-failures/osd-delay', 'msgr/async', 'msgr/async-v2only', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'openstack}', 'rados', 'rados/singleton/{all/thrash_cache_writeback_proxy_none', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}}', 'thrashers/mapgap', 'thrashosds-health', 'workloads/write_fadvise_dontneed}'] Failure: Command failed (workunit test cephadm/test_cephadm.sh) on gibba028 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=d3a78fa196cd8226153e77705dbc65cbf19a5200 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh' 2 jobs: ['5829844', '5830100'] suites intersection: ['mon_election/connectivity', 'task/test_cephadm}'] suites union: ['mon_election/connectivity', 'rados/cephadm/workunits/{distro/centos_latest', 'rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'task/test_cephadm}'] Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=nautilus 2 jobs: ['5829802', '5829990'] suites intersection: ['1-install/nautilus', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'bluestore-bitmap', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rbd-import-export', 'rbd_api}', 'readwrite', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'thrashosds-health', 'ubuntu_latest}'] suites union: ['1-install/nautilus', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'bluestore-bitmap', 'mon_election/classic', 'mon_election/connectivity', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rbd-import-export', 'rbd_api}', 'readwrite', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'thrashosds-health', 'ubuntu_latest}']