Found 366 jobs Ignoring malparsed backtrace: ceph version 16.1.0-166-gfc1e361a (fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c) pacific (rc) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe1) [0x7f40b91360ab] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x59c) [0x555e31810c4c] Ignoring malparsed backtrace: 2021-02-06T00:36:37.063 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.841454+0000", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00083392299999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.020388+0000", , 2021-02-06T00:36:37.096 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.1042000000000001e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 104.203411089, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.17825946300000001, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "throttled", , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.846130+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:08.914987+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.193 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:09.079391+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:37.256 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4760000000000001e-06 , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.277 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:39215 1.es0 1:78828434:::smithi05521660-18:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e78)", , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:36:12.056732+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.343994+0000", , 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 1.9063e-05 , 2021-02-06T00:36:37.299 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.367457+0000", , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.2547999999999998e-05 , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 44717 , 2021-02-06T00:36:37.342 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.366 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.824302+0000", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4959999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.843930+0000", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.0020290519999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "MOSDECSubOpWrite(1.fs1 47/18 ECSubWrite(tid=4985, reqid=client.4424.0:17271, at_version=47'1484, trim_to=0'0, roll_forward_to=47'1482))", Ignoring malparsed backtrace: 2021-02-06T00:36:37.827 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.430348+0000", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.3831000000000005e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.850 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.050878+0000", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019642300000000001 Ignoring malparsed backtrace: 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 108.342798449, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61213868100000002, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.442780+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.054998+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_info": { , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client": "client.4424", , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", Ignoring malparsed backtrace: 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:37.977 INFO:teuthology.orchestra.run.smithi093.stdout: "flag_point": "reached pg", , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.105779923 , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: ] Ignoring malparsed backtrace: 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.482205+0000", , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.4496e-05 , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.490276+0000", , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.000131292 , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.060 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:32790 1.as0 1:567b208f:::smithi05521660-19:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e69)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:44.196336+0000", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4800000000000007e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:33:49.401793+0000", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00059431200000000005 Ignoring malparsed backtrace: 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.653920+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.550 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.654352+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.571 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.776178+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.593 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "header_read", Ignoring malparsed backtrace: 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.616 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.637 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19615 1.7s0 1:ea563466:::smithi05521660-14:head [append 0~442368] snapc 0=[] ondisk+write+known_if_redirected e50)", , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:34:52.833245+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.837574+0000", , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4730000000000006e-06 , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.746 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.842891+0000", , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00073171300000000005 , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19619 1.7s0 1:ea563466:::smithi05521660-14:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e50)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.789 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.946122+0000", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 3.0159999999999999e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:38.811 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:19.188380+0000", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.041902981999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:39.227 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.436670+0000", , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61600527599999999 , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 18817 , 2021-02-06T00:36:39.252 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.426192+0000", , 2021-02-06T00:36:39.274 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 5.9059999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.011461+0000", , 2021-02-06T00:36:39.296 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019071299999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:39.319 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867476+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:39.342 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867785+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", , 2021-02-06T00:36:39.365 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.021659+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.386 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.409 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: ] , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } Ignoring malparsed backtrace: 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:39.587 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 2.5142000000000001e-05 , 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.617 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.644 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.6396000000000004e-05 , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.136244222 , 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.672 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: dumped all , 2021-02-06T00:36:39.705 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean , 2021-02-06T00:36:39.705 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2021-02-06T00:37:13.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 3 1 , reweighted osd.3 to 1 (10000) , 2021-02-06T00:37:14.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 3 1 Ignoring malparsed backtrace: 2021-02-06T00:37:33.496 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":125,"fsid":"6c86354a-13de-42b3-8018-2e556a88210c","created":"2021-02-06T00:33:14.600665+0000","modified":"2021-02-06T00:37:26.562048+0000","last_up_change":"2021-02-06T00:36:45.392663+0000","last_in_change":"2021-02-06T00:36:46.404363+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"unique_pool_0","create_time":"2021-02-06T00:33:38.122606+0000","flags":1,"flags_names":"hashpspool","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":true,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"3ab939df-fbe5-45cd-bb5a-abece9e67104","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":22,"up_from":91,"up_thru":0,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6817","nonce":34571},{"type":"v1","addr":"172.21.15.55:6819","nonce":34571}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6821","nonce":34571},{"type":"v1","addr":"172.21.15.55:6823","nonce":34571}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6829","nonce":34571},{"type":"v1","addr":"172.21.15.55:6830","nonce":34571}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6825","nonce":34571},{"type":"v1","addr":"172.21.15.55:6827","nonce":34571}]},"public_addr":"172.21.15.55:6819/34571","cluster_addr":"172.21.15.55:6823/34571","heartbeat_back_addr":"172.21.15.55:6830/34571","heartbeat_front_addr":"172.21.15.55:6827/34571","state":["exists","up"]},{"osd":1,"uuid":"3074b170-47fe-4c88-964b-5b22c9ebe21b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":16890},{"type":"v1","addr":"172.21.15.55:6818","nonce":16890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":16890},{"type":"v1","addr":"172.21.15.55:6822","nonce":16890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":16890},{"type":"v1","addr":"172.21.15.55:6831","nonce":16890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":16890},{"type":"v1","addr":"172.21.15.55:6826","nonce":16890}]},"public_addr":"172.21.15.55:6818/16890","cluster_addr":"172.21.15.55:6822/16890","heartbeat_back_addr":"172.21.15.55:6831/16890","heartbeat_front_addr":"172.21.15.55:6826/16890","state":["exists","up"]},{"osd":2,"uuid":"1f407f8d-c4bd-4c2b-825d-db006892f504","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":16889},{"type":"v1","addr":"172.21.15.55:6809","nonce":16889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":16889},{"type":"v1","addr":"172.21.15.55:6811","nonce":16889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":16889},{"type":"v1","addr":"172.21.15.55:6815","nonce":16889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":16889},{"type":"v1","addr":"172.21.15.55:6813","nonce":16889}]},"public_addr":"172.21.15.55:6809/16889","cluster_addr":"172.21.15.55:6811/16889","heartbeat_back_addr":"172.21.15.55:6815/16889","heartbeat_front_addr":"172.21.15.55:6813/16889","state":["exists","up"]},{"osd":3,"uuid":"2f02446d-eabb-4f93-88a5-0bf475904461","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":16891},{"type":"v1","addr":"172.21.15.55:6801","nonce":16891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":16891},{"type":"v1","addr":"172.21.15.55:6803","nonce":16891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":16891},{"type":"v1","addr":"172.21.15.55:6807","nonce":16891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":16891},{"type":"v1","addr":"172.21.15.55:6805","nonce":16891}]},"public_addr":"172.21.15.55:6801/16891","cluster_addr":"172.21.15.55:6803/16891","heartbeat_back_addr":"172.21.15.55:6807/16891","heartbeat_front_addr":"172.21.15.55:6805/16891","state":["exists","up"]},{"osd":4,"uuid":"17d16afd-4775-4c0d-bfb4-8bdf46058f68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":16855},{"type":"v1","addr":"172.21.15.93:6817","nonce":16855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":16855},{"type":"v1","addr":"172.21.15.93:6820","nonce":16855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6825","nonce":16855},{"type":"v1","addr":"172.21.15.93:6827","nonce":16855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6821","nonce":16855},{"type":"v1","addr":"172.21.15.93:6823","nonce":16855}]},"public_addr":"172.21.15.93:6817/16855","cluster_addr":"172.21.15.93:6820/16855","heartbeat_back_addr":"172.21.15.93:6827/16855","heartbeat_front_addr":"172.21.15.93:6823/16855","state":["exists","up"]},{"osd":5,"uuid":"c5503a49-e4a4-437a-8cb4-43222a82de30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":16854},{"type":"v1","addr":"172.21.15.93:6809","nonce":16854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":16854},{"type":"v1","addr":"172.21.15.93:6811","nonce":16854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":16854},{"type":"v1","addr":"172.21.15.93:6815","nonce":16854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":16854},{"type":"v1","addr":"172.21.15.93:6813","nonce":16854}]},"public_addr":"172.21.15.93:6809/16854","cluster_addr":"172.21.15.93:6811/16854","heartbeat_back_addr":"172.21.15.93:6815/16854","heartbeat_front_addr":"172.21.15.93:6813/16854","state":["exists","up"]},{"osd":6,"uuid":"407a86ed-ec3f-46dd-b55b-97816d373e01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":16856},{"type":"v1","addr":"172.21.15.93:6801","nonce":16856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":16856},{"type":"v1","addr":"172.21.15.93:6803","nonce":16856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":16856},{"type":"v1","addr":"172.21.15.93:6807","nonce":16856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":16856},{"type":"v1","addr":"172.21.15.93:6805","nonce":16856}]},"public_addr":"172.21.15.93:6801/16856","cluster_addr":"172.21.15.93:6803/16856","heartbeat_back_addr":"172.21.15.93:6807/16856","heartbeat_front_addr":"172.21.15.93:6805/16856","state":["exists","up"]},{"osd":7,"uuid":"dacec859-6dea-40cd-933a-765e0db7423e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6819","nonce":16857},{"type":"v1","addr":"172.21.15.93:6822","nonce":16857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":16857},{"type":"v1","addr":"172.21.15.93:6826","nonce":16857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":16857},{"type":"v1","addr":"172.21.15.93:6831","nonce":16857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":16857},{"type":"v1","addr":"172.21.15.93:6829","nonce":16857}]},"public_addr":"172.21.15.93:6822/16857","cluster_addr":"172.21.15.93:6826/16857","heartbeat_back_addr":"172.21.15.93:6831/16857","heartbeat_front_addr":"172.21.15.93:6829/16857","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:36:45.743199+0000","dead_epoch":23},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.676047+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.835637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.222773+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.662450+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.395082+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.967517+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:26.345206+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2021-02-06T00:37:34.512 INFO:tasks.ceph:Scrubbing osd.0 , 2021-02-06T00:37:34.513 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_ceph_fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c/qa/tasks/rados.py", line 265, in thread , run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_git_teuthology_5251f0da07972a7365332a5ff74fa4d328580822/teuthology/orchestra/run.py", line 479, in wait Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2021-02-06T00:38:09.835 INFO:tasks.ceph:Archiving crash dumps... , 2021-02-06T00:38:09.838 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2021-02-05_03:31:02-rados-pacific-distro-basic-smithi/5857765/remote/smithi055/crash Ignoring malparsed backtrace: suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ , next(self.gen) Ignoring malparsed backtrace: ceph version 16.1.0-166-gfc1e361a (fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c) pacific (rc) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe1) [0x7f40b91360ab] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x59c) [0x555e31810c4c] Ignoring malparsed backtrace: 2021-02-06T00:36:37.063 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.841454+0000", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00083392299999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.020388+0000", , 2021-02-06T00:36:37.096 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.1042000000000001e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 104.203411089, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.17825946300000001, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "throttled", , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.846130+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:08.914987+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.193 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:09.079391+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:37.256 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4760000000000001e-06 , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.277 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:39215 1.es0 1:78828434:::smithi05521660-18:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e78)", , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:36:12.056732+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.343994+0000", , 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 1.9063e-05 , 2021-02-06T00:36:37.299 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.367457+0000", , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.2547999999999998e-05 , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 44717 , 2021-02-06T00:36:37.342 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.366 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.824302+0000", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4959999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.843930+0000", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.0020290519999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "MOSDECSubOpWrite(1.fs1 47/18 ECSubWrite(tid=4985, reqid=client.4424.0:17271, at_version=47'1484, trim_to=0'0, roll_forward_to=47'1482))", Ignoring malparsed backtrace: 2021-02-06T00:36:37.827 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.430348+0000", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.3831000000000005e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.850 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.050878+0000", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019642300000000001 Ignoring malparsed backtrace: 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 108.342798449, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61213868100000002, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.442780+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.054998+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_info": { , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client": "client.4424", , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", Ignoring malparsed backtrace: 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:37.977 INFO:teuthology.orchestra.run.smithi093.stdout: "flag_point": "reached pg", , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.105779923 , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: ] Ignoring malparsed backtrace: 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.482205+0000", , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.4496e-05 , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.490276+0000", , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.000131292 , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.060 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:32790 1.as0 1:567b208f:::smithi05521660-19:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e69)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:44.196336+0000", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4800000000000007e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:33:49.401793+0000", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00059431200000000005 Ignoring malparsed backtrace: 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.653920+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.550 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.654352+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.571 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.776178+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.593 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "header_read", Ignoring malparsed backtrace: 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.616 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.637 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19615 1.7s0 1:ea563466:::smithi05521660-14:head [append 0~442368] snapc 0=[] ondisk+write+known_if_redirected e50)", , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:34:52.833245+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.837574+0000", , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4730000000000006e-06 , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.746 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.842891+0000", , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00073171300000000005 , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19619 1.7s0 1:ea563466:::smithi05521660-14:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e50)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.789 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.946122+0000", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 3.0159999999999999e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:38.811 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:19.188380+0000", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.041902981999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:39.227 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.436670+0000", , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61600527599999999 , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 18817 , 2021-02-06T00:36:39.252 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.426192+0000", , 2021-02-06T00:36:39.274 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 5.9059999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.011461+0000", , 2021-02-06T00:36:39.296 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019071299999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:39.319 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867476+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:39.342 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867785+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", , 2021-02-06T00:36:39.365 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.021659+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.386 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.409 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: ] , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } Ignoring malparsed backtrace: 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:39.587 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 2.5142000000000001e-05 , 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.617 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.644 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.6396000000000004e-05 , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.136244222 , 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.672 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: dumped all , 2021-02-06T00:36:39.705 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean , 2021-02-06T00:36:39.705 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2021-02-06T00:37:13.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 3 1 , reweighted osd.3 to 1 (10000) , 2021-02-06T00:37:14.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 3 1 Ignoring malparsed backtrace: 2021-02-06T00:37:33.496 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":125,"fsid":"6c86354a-13de-42b3-8018-2e556a88210c","created":"2021-02-06T00:33:14.600665+0000","modified":"2021-02-06T00:37:26.562048+0000","last_up_change":"2021-02-06T00:36:45.392663+0000","last_in_change":"2021-02-06T00:36:46.404363+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"unique_pool_0","create_time":"2021-02-06T00:33:38.122606+0000","flags":1,"flags_names":"hashpspool","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":true,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"3ab939df-fbe5-45cd-bb5a-abece9e67104","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":22,"up_from":91,"up_thru":0,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6817","nonce":34571},{"type":"v1","addr":"172.21.15.55:6819","nonce":34571}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6821","nonce":34571},{"type":"v1","addr":"172.21.15.55:6823","nonce":34571}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6829","nonce":34571},{"type":"v1","addr":"172.21.15.55:6830","nonce":34571}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6825","nonce":34571},{"type":"v1","addr":"172.21.15.55:6827","nonce":34571}]},"public_addr":"172.21.15.55:6819/34571","cluster_addr":"172.21.15.55:6823/34571","heartbeat_back_addr":"172.21.15.55:6830/34571","heartbeat_front_addr":"172.21.15.55:6827/34571","state":["exists","up"]},{"osd":1,"uuid":"3074b170-47fe-4c88-964b-5b22c9ebe21b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":16890},{"type":"v1","addr":"172.21.15.55:6818","nonce":16890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":16890},{"type":"v1","addr":"172.21.15.55:6822","nonce":16890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":16890},{"type":"v1","addr":"172.21.15.55:6831","nonce":16890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":16890},{"type":"v1","addr":"172.21.15.55:6826","nonce":16890}]},"public_addr":"172.21.15.55:6818/16890","cluster_addr":"172.21.15.55:6822/16890","heartbeat_back_addr":"172.21.15.55:6831/16890","heartbeat_front_addr":"172.21.15.55:6826/16890","state":["exists","up"]},{"osd":2,"uuid":"1f407f8d-c4bd-4c2b-825d-db006892f504","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":16889},{"type":"v1","addr":"172.21.15.55:6809","nonce":16889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":16889},{"type":"v1","addr":"172.21.15.55:6811","nonce":16889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":16889},{"type":"v1","addr":"172.21.15.55:6815","nonce":16889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":16889},{"type":"v1","addr":"172.21.15.55:6813","nonce":16889}]},"public_addr":"172.21.15.55:6809/16889","cluster_addr":"172.21.15.55:6811/16889","heartbeat_back_addr":"172.21.15.55:6815/16889","heartbeat_front_addr":"172.21.15.55:6813/16889","state":["exists","up"]},{"osd":3,"uuid":"2f02446d-eabb-4f93-88a5-0bf475904461","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":16891},{"type":"v1","addr":"172.21.15.55:6801","nonce":16891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":16891},{"type":"v1","addr":"172.21.15.55:6803","nonce":16891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":16891},{"type":"v1","addr":"172.21.15.55:6807","nonce":16891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":16891},{"type":"v1","addr":"172.21.15.55:6805","nonce":16891}]},"public_addr":"172.21.15.55:6801/16891","cluster_addr":"172.21.15.55:6803/16891","heartbeat_back_addr":"172.21.15.55:6807/16891","heartbeat_front_addr":"172.21.15.55:6805/16891","state":["exists","up"]},{"osd":4,"uuid":"17d16afd-4775-4c0d-bfb4-8bdf46058f68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":16855},{"type":"v1","addr":"172.21.15.93:6817","nonce":16855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":16855},{"type":"v1","addr":"172.21.15.93:6820","nonce":16855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6825","nonce":16855},{"type":"v1","addr":"172.21.15.93:6827","nonce":16855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6821","nonce":16855},{"type":"v1","addr":"172.21.15.93:6823","nonce":16855}]},"public_addr":"172.21.15.93:6817/16855","cluster_addr":"172.21.15.93:6820/16855","heartbeat_back_addr":"172.21.15.93:6827/16855","heartbeat_front_addr":"172.21.15.93:6823/16855","state":["exists","up"]},{"osd":5,"uuid":"c5503a49-e4a4-437a-8cb4-43222a82de30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":16854},{"type":"v1","addr":"172.21.15.93:6809","nonce":16854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":16854},{"type":"v1","addr":"172.21.15.93:6811","nonce":16854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":16854},{"type":"v1","addr":"172.21.15.93:6815","nonce":16854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":16854},{"type":"v1","addr":"172.21.15.93:6813","nonce":16854}]},"public_addr":"172.21.15.93:6809/16854","cluster_addr":"172.21.15.93:6811/16854","heartbeat_back_addr":"172.21.15.93:6815/16854","heartbeat_front_addr":"172.21.15.93:6813/16854","state":["exists","up"]},{"osd":6,"uuid":"407a86ed-ec3f-46dd-b55b-97816d373e01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":16856},{"type":"v1","addr":"172.21.15.93:6801","nonce":16856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":16856},{"type":"v1","addr":"172.21.15.93:6803","nonce":16856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":16856},{"type":"v1","addr":"172.21.15.93:6807","nonce":16856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":16856},{"type":"v1","addr":"172.21.15.93:6805","nonce":16856}]},"public_addr":"172.21.15.93:6801/16856","cluster_addr":"172.21.15.93:6803/16856","heartbeat_back_addr":"172.21.15.93:6807/16856","heartbeat_front_addr":"172.21.15.93:6805/16856","state":["exists","up"]},{"osd":7,"uuid":"dacec859-6dea-40cd-933a-765e0db7423e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6819","nonce":16857},{"type":"v1","addr":"172.21.15.93:6822","nonce":16857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":16857},{"type":"v1","addr":"172.21.15.93:6826","nonce":16857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":16857},{"type":"v1","addr":"172.21.15.93:6831","nonce":16857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":16857},{"type":"v1","addr":"172.21.15.93:6829","nonce":16857}]},"public_addr":"172.21.15.93:6822/16857","cluster_addr":"172.21.15.93:6826/16857","heartbeat_back_addr":"172.21.15.93:6831/16857","heartbeat_front_addr":"172.21.15.93:6829/16857","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:36:45.743199+0000","dead_epoch":23},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.676047+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.835637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.222773+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.662450+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.395082+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.967517+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:26.345206+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2021-02-06T00:37:34.512 INFO:tasks.ceph:Scrubbing osd.0 , 2021-02-06T00:37:34.513 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_ceph_fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c/qa/tasks/rados.py", line 265, in thread , run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_git_teuthology_5251f0da07972a7365332a5ff74fa4d328580822/teuthology/orchestra/run.py", line 479, in wait Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2021-02-06T00:38:09.835 INFO:tasks.ceph:Archiving crash dumps... , 2021-02-06T00:38:09.838 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2021-02-05_03:31:02-rados-pacific-distro-basic-smithi/5857765/remote/smithi055/crash Ignoring malparsed backtrace: suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ , next(self.gen) Ignoring malparsed backtrace: ceph version 16.1.0-166-gfc1e361a (fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c) pacific (rc) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe1) [0x7f40b91360ab] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x59c) [0x555e31810c4c] Ignoring malparsed backtrace: 2021-02-06T00:36:37.063 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.841454+0000", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00083392299999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.020388+0000", , 2021-02-06T00:36:37.096 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.1042000000000001e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 104.203411089, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.17825946300000001, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "throttled", , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.846130+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:08.914987+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.193 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:09.079391+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:37.256 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4760000000000001e-06 , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.277 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:39215 1.es0 1:78828434:::smithi05521660-18:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e78)", , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:36:12.056732+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.343994+0000", , 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 1.9063e-05 , 2021-02-06T00:36:37.299 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.367457+0000", , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.2547999999999998e-05 , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 44717 , 2021-02-06T00:36:37.342 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.366 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.824302+0000", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4959999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.843930+0000", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.0020290519999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "MOSDECSubOpWrite(1.fs1 47/18 ECSubWrite(tid=4985, reqid=client.4424.0:17271, at_version=47'1484, trim_to=0'0, roll_forward_to=47'1482))", Ignoring malparsed backtrace: 2021-02-06T00:36:37.827 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.430348+0000", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.3831000000000005e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.850 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.050878+0000", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019642300000000001 Ignoring malparsed backtrace: 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 108.342798449, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61213868100000002, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.442780+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.054998+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_info": { , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client": "client.4424", , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", Ignoring malparsed backtrace: 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:37.977 INFO:teuthology.orchestra.run.smithi093.stdout: "flag_point": "reached pg", , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.105779923 , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: ] Ignoring malparsed backtrace: 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.482205+0000", , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.4496e-05 , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.490276+0000", , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.000131292 , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.060 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:32790 1.as0 1:567b208f:::smithi05521660-19:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e69)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:44.196336+0000", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4800000000000007e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:33:49.401793+0000", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00059431200000000005 Ignoring malparsed backtrace: 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.653920+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.550 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.654352+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.571 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.776178+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.593 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "header_read", Ignoring malparsed backtrace: 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.616 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.637 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19615 1.7s0 1:ea563466:::smithi05521660-14:head [append 0~442368] snapc 0=[] ondisk+write+known_if_redirected e50)", , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:34:52.833245+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.837574+0000", , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4730000000000006e-06 , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.746 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.842891+0000", , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00073171300000000005 , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19619 1.7s0 1:ea563466:::smithi05521660-14:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e50)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.789 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.946122+0000", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 3.0159999999999999e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:38.811 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:19.188380+0000", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.041902981999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:39.227 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.436670+0000", , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61600527599999999 , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 18817 , 2021-02-06T00:36:39.252 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.426192+0000", , 2021-02-06T00:36:39.274 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 5.9059999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.011461+0000", , 2021-02-06T00:36:39.296 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019071299999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:39.319 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867476+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:39.342 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867785+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", , 2021-02-06T00:36:39.365 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.021659+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.386 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.409 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: ] , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } Ignoring malparsed backtrace: 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:39.587 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 2.5142000000000001e-05 , 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.617 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.644 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.6396000000000004e-05 , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.136244222 , 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.672 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: dumped all , 2021-02-06T00:36:39.705 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean , 2021-02-06T00:36:39.705 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2021-02-06T00:37:13.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 3 1 , reweighted osd.3 to 1 (10000) , 2021-02-06T00:37:14.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 3 1 Ignoring malparsed backtrace: 2021-02-06T00:37:33.496 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":125,"fsid":"6c86354a-13de-42b3-8018-2e556a88210c","created":"2021-02-06T00:33:14.600665+0000","modified":"2021-02-06T00:37:26.562048+0000","last_up_change":"2021-02-06T00:36:45.392663+0000","last_in_change":"2021-02-06T00:36:46.404363+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"unique_pool_0","create_time":"2021-02-06T00:33:38.122606+0000","flags":1,"flags_names":"hashpspool","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":true,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"3ab939df-fbe5-45cd-bb5a-abece9e67104","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":22,"up_from":91,"up_thru":0,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6817","nonce":34571},{"type":"v1","addr":"172.21.15.55:6819","nonce":34571}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6821","nonce":34571},{"type":"v1","addr":"172.21.15.55:6823","nonce":34571}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6829","nonce":34571},{"type":"v1","addr":"172.21.15.55:6830","nonce":34571}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6825","nonce":34571},{"type":"v1","addr":"172.21.15.55:6827","nonce":34571}]},"public_addr":"172.21.15.55:6819/34571","cluster_addr":"172.21.15.55:6823/34571","heartbeat_back_addr":"172.21.15.55:6830/34571","heartbeat_front_addr":"172.21.15.55:6827/34571","state":["exists","up"]},{"osd":1,"uuid":"3074b170-47fe-4c88-964b-5b22c9ebe21b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":16890},{"type":"v1","addr":"172.21.15.55:6818","nonce":16890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":16890},{"type":"v1","addr":"172.21.15.55:6822","nonce":16890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":16890},{"type":"v1","addr":"172.21.15.55:6831","nonce":16890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":16890},{"type":"v1","addr":"172.21.15.55:6826","nonce":16890}]},"public_addr":"172.21.15.55:6818/16890","cluster_addr":"172.21.15.55:6822/16890","heartbeat_back_addr":"172.21.15.55:6831/16890","heartbeat_front_addr":"172.21.15.55:6826/16890","state":["exists","up"]},{"osd":2,"uuid":"1f407f8d-c4bd-4c2b-825d-db006892f504","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":16889},{"type":"v1","addr":"172.21.15.55:6809","nonce":16889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":16889},{"type":"v1","addr":"172.21.15.55:6811","nonce":16889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":16889},{"type":"v1","addr":"172.21.15.55:6815","nonce":16889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":16889},{"type":"v1","addr":"172.21.15.55:6813","nonce":16889}]},"public_addr":"172.21.15.55:6809/16889","cluster_addr":"172.21.15.55:6811/16889","heartbeat_back_addr":"172.21.15.55:6815/16889","heartbeat_front_addr":"172.21.15.55:6813/16889","state":["exists","up"]},{"osd":3,"uuid":"2f02446d-eabb-4f93-88a5-0bf475904461","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":16891},{"type":"v1","addr":"172.21.15.55:6801","nonce":16891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":16891},{"type":"v1","addr":"172.21.15.55:6803","nonce":16891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":16891},{"type":"v1","addr":"172.21.15.55:6807","nonce":16891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":16891},{"type":"v1","addr":"172.21.15.55:6805","nonce":16891}]},"public_addr":"172.21.15.55:6801/16891","cluster_addr":"172.21.15.55:6803/16891","heartbeat_back_addr":"172.21.15.55:6807/16891","heartbeat_front_addr":"172.21.15.55:6805/16891","state":["exists","up"]},{"osd":4,"uuid":"17d16afd-4775-4c0d-bfb4-8bdf46058f68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":16855},{"type":"v1","addr":"172.21.15.93:6817","nonce":16855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":16855},{"type":"v1","addr":"172.21.15.93:6820","nonce":16855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6825","nonce":16855},{"type":"v1","addr":"172.21.15.93:6827","nonce":16855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6821","nonce":16855},{"type":"v1","addr":"172.21.15.93:6823","nonce":16855}]},"public_addr":"172.21.15.93:6817/16855","cluster_addr":"172.21.15.93:6820/16855","heartbeat_back_addr":"172.21.15.93:6827/16855","heartbeat_front_addr":"172.21.15.93:6823/16855","state":["exists","up"]},{"osd":5,"uuid":"c5503a49-e4a4-437a-8cb4-43222a82de30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":16854},{"type":"v1","addr":"172.21.15.93:6809","nonce":16854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":16854},{"type":"v1","addr":"172.21.15.93:6811","nonce":16854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":16854},{"type":"v1","addr":"172.21.15.93:6815","nonce":16854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":16854},{"type":"v1","addr":"172.21.15.93:6813","nonce":16854}]},"public_addr":"172.21.15.93:6809/16854","cluster_addr":"172.21.15.93:6811/16854","heartbeat_back_addr":"172.21.15.93:6815/16854","heartbeat_front_addr":"172.21.15.93:6813/16854","state":["exists","up"]},{"osd":6,"uuid":"407a86ed-ec3f-46dd-b55b-97816d373e01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":16856},{"type":"v1","addr":"172.21.15.93:6801","nonce":16856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":16856},{"type":"v1","addr":"172.21.15.93:6803","nonce":16856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":16856},{"type":"v1","addr":"172.21.15.93:6807","nonce":16856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":16856},{"type":"v1","addr":"172.21.15.93:6805","nonce":16856}]},"public_addr":"172.21.15.93:6801/16856","cluster_addr":"172.21.15.93:6803/16856","heartbeat_back_addr":"172.21.15.93:6807/16856","heartbeat_front_addr":"172.21.15.93:6805/16856","state":["exists","up"]},{"osd":7,"uuid":"dacec859-6dea-40cd-933a-765e0db7423e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6819","nonce":16857},{"type":"v1","addr":"172.21.15.93:6822","nonce":16857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":16857},{"type":"v1","addr":"172.21.15.93:6826","nonce":16857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":16857},{"type":"v1","addr":"172.21.15.93:6831","nonce":16857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":16857},{"type":"v1","addr":"172.21.15.93:6829","nonce":16857}]},"public_addr":"172.21.15.93:6822/16857","cluster_addr":"172.21.15.93:6826/16857","heartbeat_back_addr":"172.21.15.93:6831/16857","heartbeat_front_addr":"172.21.15.93:6829/16857","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:36:45.743199+0000","dead_epoch":23},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.676047+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.835637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.222773+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.662450+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.395082+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.967517+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:26.345206+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2021-02-06T00:37:34.512 INFO:tasks.ceph:Scrubbing osd.0 , 2021-02-06T00:37:34.513 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_ceph_fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c/qa/tasks/rados.py", line 265, in thread , run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_git_teuthology_5251f0da07972a7365332a5ff74fa4d328580822/teuthology/orchestra/run.py", line 479, in wait Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2021-02-06T00:38:09.835 INFO:tasks.ceph:Archiving crash dumps... , 2021-02-06T00:38:09.838 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2021-02-05_03:31:02-rados-pacific-distro-basic-smithi/5857765/remote/smithi055/crash Ignoring malparsed backtrace: suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ , next(self.gen) Ignoring malparsed backtrace: ceph version 16.1.0-166-gfc1e361a (fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c) pacific (rc) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe1) [0x7f40b91360ab] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x59c) [0x555e31810c4c] Ignoring malparsed backtrace: 2021-02-06T00:36:37.063 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.841454+0000", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00083392299999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.020388+0000", , 2021-02-06T00:36:37.096 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.1042000000000001e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 104.203411089, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.17825946300000001, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "throttled", , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.846130+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:08.914987+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.193 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:09.079391+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:37.256 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4760000000000001e-06 , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.277 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:39215 1.es0 1:78828434:::smithi05521660-18:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e78)", , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:36:12.056732+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.343994+0000", , 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 1.9063e-05 , 2021-02-06T00:36:37.299 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.367457+0000", , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.2547999999999998e-05 , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 44717 , 2021-02-06T00:36:37.342 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.366 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.824302+0000", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4959999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.843930+0000", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.0020290519999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "MOSDECSubOpWrite(1.fs1 47/18 ECSubWrite(tid=4985, reqid=client.4424.0:17271, at_version=47'1484, trim_to=0'0, roll_forward_to=47'1482))", Ignoring malparsed backtrace: 2021-02-06T00:36:37.827 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.430348+0000", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.3831000000000005e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.850 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.050878+0000", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019642300000000001 Ignoring malparsed backtrace: 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 108.342798449, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61213868100000002, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.442780+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.054998+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_info": { , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client": "client.4424", , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", Ignoring malparsed backtrace: 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:37.977 INFO:teuthology.orchestra.run.smithi093.stdout: "flag_point": "reached pg", , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.105779923 , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: ] Ignoring malparsed backtrace: 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.482205+0000", , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.4496e-05 , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.490276+0000", , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.000131292 , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.060 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:32790 1.as0 1:567b208f:::smithi05521660-19:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e69)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:44.196336+0000", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4800000000000007e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:33:49.401793+0000", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00059431200000000005 Ignoring malparsed backtrace: 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.653920+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.550 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.654352+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.571 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.776178+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.593 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "header_read", Ignoring malparsed backtrace: 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.616 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.637 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19615 1.7s0 1:ea563466:::smithi05521660-14:head [append 0~442368] snapc 0=[] ondisk+write+known_if_redirected e50)", , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:34:52.833245+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.837574+0000", , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4730000000000006e-06 , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.746 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.842891+0000", , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00073171300000000005 , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19619 1.7s0 1:ea563466:::smithi05521660-14:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e50)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.789 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.946122+0000", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 3.0159999999999999e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:38.811 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:19.188380+0000", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.041902981999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:39.227 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.436670+0000", , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61600527599999999 , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 18817 , 2021-02-06T00:36:39.252 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.426192+0000", , 2021-02-06T00:36:39.274 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 5.9059999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.011461+0000", , 2021-02-06T00:36:39.296 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019071299999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:39.319 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867476+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:39.342 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867785+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", , 2021-02-06T00:36:39.365 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.021659+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.386 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.409 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: ] , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } Ignoring malparsed backtrace: 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:39.587 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 2.5142000000000001e-05 , 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.617 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.644 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.6396000000000004e-05 , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.136244222 , 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.672 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: dumped all , 2021-02-06T00:36:39.705 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean , 2021-02-06T00:36:39.705 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2021-02-06T00:37:13.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 3 1 , reweighted osd.3 to 1 (10000) , 2021-02-06T00:37:14.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 3 1 Ignoring malparsed backtrace: 2021-02-06T00:37:33.496 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":125,"fsid":"6c86354a-13de-42b3-8018-2e556a88210c","created":"2021-02-06T00:33:14.600665+0000","modified":"2021-02-06T00:37:26.562048+0000","last_up_change":"2021-02-06T00:36:45.392663+0000","last_in_change":"2021-02-06T00:36:46.404363+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"unique_pool_0","create_time":"2021-02-06T00:33:38.122606+0000","flags":1,"flags_names":"hashpspool","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":true,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"3ab939df-fbe5-45cd-bb5a-abece9e67104","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":22,"up_from":91,"up_thru":0,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6817","nonce":34571},{"type":"v1","addr":"172.21.15.55:6819","nonce":34571}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6821","nonce":34571},{"type":"v1","addr":"172.21.15.55:6823","nonce":34571}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6829","nonce":34571},{"type":"v1","addr":"172.21.15.55:6830","nonce":34571}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6825","nonce":34571},{"type":"v1","addr":"172.21.15.55:6827","nonce":34571}]},"public_addr":"172.21.15.55:6819/34571","cluster_addr":"172.21.15.55:6823/34571","heartbeat_back_addr":"172.21.15.55:6830/34571","heartbeat_front_addr":"172.21.15.55:6827/34571","state":["exists","up"]},{"osd":1,"uuid":"3074b170-47fe-4c88-964b-5b22c9ebe21b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":16890},{"type":"v1","addr":"172.21.15.55:6818","nonce":16890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":16890},{"type":"v1","addr":"172.21.15.55:6822","nonce":16890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":16890},{"type":"v1","addr":"172.21.15.55:6831","nonce":16890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":16890},{"type":"v1","addr":"172.21.15.55:6826","nonce":16890}]},"public_addr":"172.21.15.55:6818/16890","cluster_addr":"172.21.15.55:6822/16890","heartbeat_back_addr":"172.21.15.55:6831/16890","heartbeat_front_addr":"172.21.15.55:6826/16890","state":["exists","up"]},{"osd":2,"uuid":"1f407f8d-c4bd-4c2b-825d-db006892f504","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":16889},{"type":"v1","addr":"172.21.15.55:6809","nonce":16889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":16889},{"type":"v1","addr":"172.21.15.55:6811","nonce":16889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":16889},{"type":"v1","addr":"172.21.15.55:6815","nonce":16889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":16889},{"type":"v1","addr":"172.21.15.55:6813","nonce":16889}]},"public_addr":"172.21.15.55:6809/16889","cluster_addr":"172.21.15.55:6811/16889","heartbeat_back_addr":"172.21.15.55:6815/16889","heartbeat_front_addr":"172.21.15.55:6813/16889","state":["exists","up"]},{"osd":3,"uuid":"2f02446d-eabb-4f93-88a5-0bf475904461","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":16891},{"type":"v1","addr":"172.21.15.55:6801","nonce":16891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":16891},{"type":"v1","addr":"172.21.15.55:6803","nonce":16891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":16891},{"type":"v1","addr":"172.21.15.55:6807","nonce":16891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":16891},{"type":"v1","addr":"172.21.15.55:6805","nonce":16891}]},"public_addr":"172.21.15.55:6801/16891","cluster_addr":"172.21.15.55:6803/16891","heartbeat_back_addr":"172.21.15.55:6807/16891","heartbeat_front_addr":"172.21.15.55:6805/16891","state":["exists","up"]},{"osd":4,"uuid":"17d16afd-4775-4c0d-bfb4-8bdf46058f68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":16855},{"type":"v1","addr":"172.21.15.93:6817","nonce":16855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":16855},{"type":"v1","addr":"172.21.15.93:6820","nonce":16855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6825","nonce":16855},{"type":"v1","addr":"172.21.15.93:6827","nonce":16855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6821","nonce":16855},{"type":"v1","addr":"172.21.15.93:6823","nonce":16855}]},"public_addr":"172.21.15.93:6817/16855","cluster_addr":"172.21.15.93:6820/16855","heartbeat_back_addr":"172.21.15.93:6827/16855","heartbeat_front_addr":"172.21.15.93:6823/16855","state":["exists","up"]},{"osd":5,"uuid":"c5503a49-e4a4-437a-8cb4-43222a82de30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":16854},{"type":"v1","addr":"172.21.15.93:6809","nonce":16854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":16854},{"type":"v1","addr":"172.21.15.93:6811","nonce":16854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":16854},{"type":"v1","addr":"172.21.15.93:6815","nonce":16854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":16854},{"type":"v1","addr":"172.21.15.93:6813","nonce":16854}]},"public_addr":"172.21.15.93:6809/16854","cluster_addr":"172.21.15.93:6811/16854","heartbeat_back_addr":"172.21.15.93:6815/16854","heartbeat_front_addr":"172.21.15.93:6813/16854","state":["exists","up"]},{"osd":6,"uuid":"407a86ed-ec3f-46dd-b55b-97816d373e01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":16856},{"type":"v1","addr":"172.21.15.93:6801","nonce":16856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":16856},{"type":"v1","addr":"172.21.15.93:6803","nonce":16856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":16856},{"type":"v1","addr":"172.21.15.93:6807","nonce":16856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":16856},{"type":"v1","addr":"172.21.15.93:6805","nonce":16856}]},"public_addr":"172.21.15.93:6801/16856","cluster_addr":"172.21.15.93:6803/16856","heartbeat_back_addr":"172.21.15.93:6807/16856","heartbeat_front_addr":"172.21.15.93:6805/16856","state":["exists","up"]},{"osd":7,"uuid":"dacec859-6dea-40cd-933a-765e0db7423e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6819","nonce":16857},{"type":"v1","addr":"172.21.15.93:6822","nonce":16857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":16857},{"type":"v1","addr":"172.21.15.93:6826","nonce":16857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":16857},{"type":"v1","addr":"172.21.15.93:6831","nonce":16857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":16857},{"type":"v1","addr":"172.21.15.93:6829","nonce":16857}]},"public_addr":"172.21.15.93:6822/16857","cluster_addr":"172.21.15.93:6826/16857","heartbeat_back_addr":"172.21.15.93:6831/16857","heartbeat_front_addr":"172.21.15.93:6829/16857","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:36:45.743199+0000","dead_epoch":23},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.676047+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.835637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.222773+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.662450+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.395082+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.967517+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:26.345206+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2021-02-06T00:37:34.512 INFO:tasks.ceph:Scrubbing osd.0 , 2021-02-06T00:37:34.513 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_ceph_fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c/qa/tasks/rados.py", line 265, in thread , run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_git_teuthology_5251f0da07972a7365332a5ff74fa4d328580822/teuthology/orchestra/run.py", line 479, in wait Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2021-02-06T00:38:09.835 INFO:tasks.ceph:Archiving crash dumps... , 2021-02-06T00:38:09.838 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2021-02-05_03:31:02-rados-pacific-distro-basic-smithi/5857765/remote/smithi055/crash Ignoring malparsed backtrace: suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ , next(self.gen) Ignoring malparsed backtrace: ceph version 16.1.0-166-gfc1e361a (fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c) pacific (rc) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe1) [0x7f40b91360ab] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x59c) [0x555e31810c4c] Ignoring malparsed backtrace: 2021-02-06T00:36:37.063 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.841454+0000", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00083392299999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.020388+0000", , 2021-02-06T00:36:37.096 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.1042000000000001e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 104.203411089, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.17825946300000001, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "throttled", , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.846130+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:08.914987+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.193 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:09.079391+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:37.256 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4760000000000001e-06 , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.277 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:39215 1.es0 1:78828434:::smithi05521660-18:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e78)", , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:36:12.056732+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.343994+0000", , 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 1.9063e-05 , 2021-02-06T00:36:37.299 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.367457+0000", , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.2547999999999998e-05 , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 44717 , 2021-02-06T00:36:37.342 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.366 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.824302+0000", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4959999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.843930+0000", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.0020290519999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "MOSDECSubOpWrite(1.fs1 47/18 ECSubWrite(tid=4985, reqid=client.4424.0:17271, at_version=47'1484, trim_to=0'0, roll_forward_to=47'1482))", Ignoring malparsed backtrace: 2021-02-06T00:36:37.827 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.430348+0000", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.3831000000000005e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.850 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.050878+0000", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019642300000000001 Ignoring malparsed backtrace: 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 108.342798449, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61213868100000002, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.442780+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.054998+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_info": { , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client": "client.4424", , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", Ignoring malparsed backtrace: 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:37.977 INFO:teuthology.orchestra.run.smithi093.stdout: "flag_point": "reached pg", , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.105779923 , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: ] Ignoring malparsed backtrace: 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.482205+0000", , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.4496e-05 , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.490276+0000", , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.000131292 , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.060 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:32790 1.as0 1:567b208f:::smithi05521660-19:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e69)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:44.196336+0000", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4800000000000007e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:33:49.401793+0000", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00059431200000000005 Ignoring malparsed backtrace: 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.653920+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.550 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.654352+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.571 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.776178+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.593 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "header_read", Ignoring malparsed backtrace: 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.616 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.637 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19615 1.7s0 1:ea563466:::smithi05521660-14:head [append 0~442368] snapc 0=[] ondisk+write+known_if_redirected e50)", , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:34:52.833245+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.837574+0000", , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4730000000000006e-06 , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.746 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.842891+0000", , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00073171300000000005 , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19619 1.7s0 1:ea563466:::smithi05521660-14:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e50)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.789 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.946122+0000", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 3.0159999999999999e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:38.811 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:19.188380+0000", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.041902981999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:39.227 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.436670+0000", , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61600527599999999 , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 18817 , 2021-02-06T00:36:39.252 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.426192+0000", , 2021-02-06T00:36:39.274 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 5.9059999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.011461+0000", , 2021-02-06T00:36:39.296 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019071299999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:39.319 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867476+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:39.342 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867785+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", , 2021-02-06T00:36:39.365 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.021659+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.386 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.409 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: ] , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } Ignoring malparsed backtrace: 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:39.587 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 2.5142000000000001e-05 , 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.617 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.644 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.6396000000000004e-05 , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.136244222 , 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.672 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: dumped all , 2021-02-06T00:36:39.705 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean , 2021-02-06T00:36:39.705 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2021-02-06T00:37:13.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 3 1 , reweighted osd.3 to 1 (10000) , 2021-02-06T00:37:14.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 3 1 Ignoring malparsed backtrace: 2021-02-06T00:37:33.496 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":125,"fsid":"6c86354a-13de-42b3-8018-2e556a88210c","created":"2021-02-06T00:33:14.600665+0000","modified":"2021-02-06T00:37:26.562048+0000","last_up_change":"2021-02-06T00:36:45.392663+0000","last_in_change":"2021-02-06T00:36:46.404363+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"unique_pool_0","create_time":"2021-02-06T00:33:38.122606+0000","flags":1,"flags_names":"hashpspool","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":true,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"3ab939df-fbe5-45cd-bb5a-abece9e67104","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":22,"up_from":91,"up_thru":0,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6817","nonce":34571},{"type":"v1","addr":"172.21.15.55:6819","nonce":34571}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6821","nonce":34571},{"type":"v1","addr":"172.21.15.55:6823","nonce":34571}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6829","nonce":34571},{"type":"v1","addr":"172.21.15.55:6830","nonce":34571}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6825","nonce":34571},{"type":"v1","addr":"172.21.15.55:6827","nonce":34571}]},"public_addr":"172.21.15.55:6819/34571","cluster_addr":"172.21.15.55:6823/34571","heartbeat_back_addr":"172.21.15.55:6830/34571","heartbeat_front_addr":"172.21.15.55:6827/34571","state":["exists","up"]},{"osd":1,"uuid":"3074b170-47fe-4c88-964b-5b22c9ebe21b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":16890},{"type":"v1","addr":"172.21.15.55:6818","nonce":16890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":16890},{"type":"v1","addr":"172.21.15.55:6822","nonce":16890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":16890},{"type":"v1","addr":"172.21.15.55:6831","nonce":16890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":16890},{"type":"v1","addr":"172.21.15.55:6826","nonce":16890}]},"public_addr":"172.21.15.55:6818/16890","cluster_addr":"172.21.15.55:6822/16890","heartbeat_back_addr":"172.21.15.55:6831/16890","heartbeat_front_addr":"172.21.15.55:6826/16890","state":["exists","up"]},{"osd":2,"uuid":"1f407f8d-c4bd-4c2b-825d-db006892f504","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":16889},{"type":"v1","addr":"172.21.15.55:6809","nonce":16889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":16889},{"type":"v1","addr":"172.21.15.55:6811","nonce":16889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":16889},{"type":"v1","addr":"172.21.15.55:6815","nonce":16889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":16889},{"type":"v1","addr":"172.21.15.55:6813","nonce":16889}]},"public_addr":"172.21.15.55:6809/16889","cluster_addr":"172.21.15.55:6811/16889","heartbeat_back_addr":"172.21.15.55:6815/16889","heartbeat_front_addr":"172.21.15.55:6813/16889","state":["exists","up"]},{"osd":3,"uuid":"2f02446d-eabb-4f93-88a5-0bf475904461","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":16891},{"type":"v1","addr":"172.21.15.55:6801","nonce":16891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":16891},{"type":"v1","addr":"172.21.15.55:6803","nonce":16891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":16891},{"type":"v1","addr":"172.21.15.55:6807","nonce":16891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":16891},{"type":"v1","addr":"172.21.15.55:6805","nonce":16891}]},"public_addr":"172.21.15.55:6801/16891","cluster_addr":"172.21.15.55:6803/16891","heartbeat_back_addr":"172.21.15.55:6807/16891","heartbeat_front_addr":"172.21.15.55:6805/16891","state":["exists","up"]},{"osd":4,"uuid":"17d16afd-4775-4c0d-bfb4-8bdf46058f68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":16855},{"type":"v1","addr":"172.21.15.93:6817","nonce":16855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":16855},{"type":"v1","addr":"172.21.15.93:6820","nonce":16855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6825","nonce":16855},{"type":"v1","addr":"172.21.15.93:6827","nonce":16855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6821","nonce":16855},{"type":"v1","addr":"172.21.15.93:6823","nonce":16855}]},"public_addr":"172.21.15.93:6817/16855","cluster_addr":"172.21.15.93:6820/16855","heartbeat_back_addr":"172.21.15.93:6827/16855","heartbeat_front_addr":"172.21.15.93:6823/16855","state":["exists","up"]},{"osd":5,"uuid":"c5503a49-e4a4-437a-8cb4-43222a82de30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":16854},{"type":"v1","addr":"172.21.15.93:6809","nonce":16854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":16854},{"type":"v1","addr":"172.21.15.93:6811","nonce":16854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":16854},{"type":"v1","addr":"172.21.15.93:6815","nonce":16854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":16854},{"type":"v1","addr":"172.21.15.93:6813","nonce":16854}]},"public_addr":"172.21.15.93:6809/16854","cluster_addr":"172.21.15.93:6811/16854","heartbeat_back_addr":"172.21.15.93:6815/16854","heartbeat_front_addr":"172.21.15.93:6813/16854","state":["exists","up"]},{"osd":6,"uuid":"407a86ed-ec3f-46dd-b55b-97816d373e01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":16856},{"type":"v1","addr":"172.21.15.93:6801","nonce":16856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":16856},{"type":"v1","addr":"172.21.15.93:6803","nonce":16856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":16856},{"type":"v1","addr":"172.21.15.93:6807","nonce":16856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":16856},{"type":"v1","addr":"172.21.15.93:6805","nonce":16856}]},"public_addr":"172.21.15.93:6801/16856","cluster_addr":"172.21.15.93:6803/16856","heartbeat_back_addr":"172.21.15.93:6807/16856","heartbeat_front_addr":"172.21.15.93:6805/16856","state":["exists","up"]},{"osd":7,"uuid":"dacec859-6dea-40cd-933a-765e0db7423e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6819","nonce":16857},{"type":"v1","addr":"172.21.15.93:6822","nonce":16857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":16857},{"type":"v1","addr":"172.21.15.93:6826","nonce":16857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":16857},{"type":"v1","addr":"172.21.15.93:6831","nonce":16857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":16857},{"type":"v1","addr":"172.21.15.93:6829","nonce":16857}]},"public_addr":"172.21.15.93:6822/16857","cluster_addr":"172.21.15.93:6826/16857","heartbeat_back_addr":"172.21.15.93:6831/16857","heartbeat_front_addr":"172.21.15.93:6829/16857","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:36:45.743199+0000","dead_epoch":23},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.676047+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.835637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.222773+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.662450+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.395082+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.967517+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:26.345206+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2021-02-06T00:37:34.512 INFO:tasks.ceph:Scrubbing osd.0 , 2021-02-06T00:37:34.513 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_ceph_fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c/qa/tasks/rados.py", line 265, in thread , run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_git_teuthology_5251f0da07972a7365332a5ff74fa4d328580822/teuthology/orchestra/run.py", line 479, in wait Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2021-02-06T00:38:09.835 INFO:tasks.ceph:Archiving crash dumps... , 2021-02-06T00:38:09.838 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2021-02-05_03:31:02-rados-pacific-distro-basic-smithi/5857765/remote/smithi055/crash Ignoring malparsed backtrace: suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ , next(self.gen) Ignoring malparsed backtrace: ceph version 16.1.0-166-gfc1e361a (fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c) pacific (rc) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe1) [0x7f40b91360ab] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x59c) [0x555e31810c4c] Ignoring malparsed backtrace: 2021-02-06T00:36:37.063 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.841454+0000", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00083392299999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.020388+0000", , 2021-02-06T00:36:37.096 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.1042000000000001e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 104.203411089, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.17825946300000001, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "throttled", , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.846130+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:08.914987+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.193 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:09.079391+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:37.256 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4760000000000001e-06 , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.277 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:39215 1.es0 1:78828434:::smithi05521660-18:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e78)", , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:36:12.056732+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.343994+0000", , 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 1.9063e-05 , 2021-02-06T00:36:37.299 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.367457+0000", , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.2547999999999998e-05 , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 44717 , 2021-02-06T00:36:37.342 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.366 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.824302+0000", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4959999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.843930+0000", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.0020290519999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "MOSDECSubOpWrite(1.fs1 47/18 ECSubWrite(tid=4985, reqid=client.4424.0:17271, at_version=47'1484, trim_to=0'0, roll_forward_to=47'1482))", Ignoring malparsed backtrace: 2021-02-06T00:36:37.827 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.430348+0000", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.3831000000000005e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.850 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.050878+0000", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019642300000000001 Ignoring malparsed backtrace: 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 108.342798449, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61213868100000002, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.442780+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.054998+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_info": { , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client": "client.4424", , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", Ignoring malparsed backtrace: 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:37.977 INFO:teuthology.orchestra.run.smithi093.stdout: "flag_point": "reached pg", , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.105779923 , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: ] Ignoring malparsed backtrace: 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.482205+0000", , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.4496e-05 , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.490276+0000", , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.000131292 , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.060 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:32790 1.as0 1:567b208f:::smithi05521660-19:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e69)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:44.196336+0000", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4800000000000007e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:33:49.401793+0000", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00059431200000000005 Ignoring malparsed backtrace: 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.653920+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.550 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.654352+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.571 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.776178+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.593 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "header_read", Ignoring malparsed backtrace: 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.616 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.637 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19615 1.7s0 1:ea563466:::smithi05521660-14:head [append 0~442368] snapc 0=[] ondisk+write+known_if_redirected e50)", , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:34:52.833245+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.837574+0000", , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4730000000000006e-06 , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.746 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.842891+0000", , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00073171300000000005 , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19619 1.7s0 1:ea563466:::smithi05521660-14:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e50)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.789 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.946122+0000", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 3.0159999999999999e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:38.811 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:19.188380+0000", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.041902981999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:39.227 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.436670+0000", , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61600527599999999 , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 18817 , 2021-02-06T00:36:39.252 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.426192+0000", , 2021-02-06T00:36:39.274 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 5.9059999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.011461+0000", , 2021-02-06T00:36:39.296 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019071299999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:39.319 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867476+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:39.342 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867785+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", , 2021-02-06T00:36:39.365 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.021659+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.386 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.409 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: ] , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } Ignoring malparsed backtrace: 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:39.587 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 2.5142000000000001e-05 , 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.617 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.644 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.6396000000000004e-05 , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.136244222 , 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.672 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: dumped all , 2021-02-06T00:36:39.705 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean , 2021-02-06T00:36:39.705 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2021-02-06T00:37:13.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 3 1 , reweighted osd.3 to 1 (10000) , 2021-02-06T00:37:14.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 3 1 Ignoring malparsed backtrace: 2021-02-06T00:37:33.496 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":125,"fsid":"6c86354a-13de-42b3-8018-2e556a88210c","created":"2021-02-06T00:33:14.600665+0000","modified":"2021-02-06T00:37:26.562048+0000","last_up_change":"2021-02-06T00:36:45.392663+0000","last_in_change":"2021-02-06T00:36:46.404363+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"unique_pool_0","create_time":"2021-02-06T00:33:38.122606+0000","flags":1,"flags_names":"hashpspool","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":true,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"3ab939df-fbe5-45cd-bb5a-abece9e67104","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":22,"up_from":91,"up_thru":0,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6817","nonce":34571},{"type":"v1","addr":"172.21.15.55:6819","nonce":34571}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6821","nonce":34571},{"type":"v1","addr":"172.21.15.55:6823","nonce":34571}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6829","nonce":34571},{"type":"v1","addr":"172.21.15.55:6830","nonce":34571}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6825","nonce":34571},{"type":"v1","addr":"172.21.15.55:6827","nonce":34571}]},"public_addr":"172.21.15.55:6819/34571","cluster_addr":"172.21.15.55:6823/34571","heartbeat_back_addr":"172.21.15.55:6830/34571","heartbeat_front_addr":"172.21.15.55:6827/34571","state":["exists","up"]},{"osd":1,"uuid":"3074b170-47fe-4c88-964b-5b22c9ebe21b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":16890},{"type":"v1","addr":"172.21.15.55:6818","nonce":16890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":16890},{"type":"v1","addr":"172.21.15.55:6822","nonce":16890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":16890},{"type":"v1","addr":"172.21.15.55:6831","nonce":16890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":16890},{"type":"v1","addr":"172.21.15.55:6826","nonce":16890}]},"public_addr":"172.21.15.55:6818/16890","cluster_addr":"172.21.15.55:6822/16890","heartbeat_back_addr":"172.21.15.55:6831/16890","heartbeat_front_addr":"172.21.15.55:6826/16890","state":["exists","up"]},{"osd":2,"uuid":"1f407f8d-c4bd-4c2b-825d-db006892f504","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":16889},{"type":"v1","addr":"172.21.15.55:6809","nonce":16889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":16889},{"type":"v1","addr":"172.21.15.55:6811","nonce":16889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":16889},{"type":"v1","addr":"172.21.15.55:6815","nonce":16889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":16889},{"type":"v1","addr":"172.21.15.55:6813","nonce":16889}]},"public_addr":"172.21.15.55:6809/16889","cluster_addr":"172.21.15.55:6811/16889","heartbeat_back_addr":"172.21.15.55:6815/16889","heartbeat_front_addr":"172.21.15.55:6813/16889","state":["exists","up"]},{"osd":3,"uuid":"2f02446d-eabb-4f93-88a5-0bf475904461","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":16891},{"type":"v1","addr":"172.21.15.55:6801","nonce":16891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":16891},{"type":"v1","addr":"172.21.15.55:6803","nonce":16891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":16891},{"type":"v1","addr":"172.21.15.55:6807","nonce":16891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":16891},{"type":"v1","addr":"172.21.15.55:6805","nonce":16891}]},"public_addr":"172.21.15.55:6801/16891","cluster_addr":"172.21.15.55:6803/16891","heartbeat_back_addr":"172.21.15.55:6807/16891","heartbeat_front_addr":"172.21.15.55:6805/16891","state":["exists","up"]},{"osd":4,"uuid":"17d16afd-4775-4c0d-bfb4-8bdf46058f68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":16855},{"type":"v1","addr":"172.21.15.93:6817","nonce":16855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":16855},{"type":"v1","addr":"172.21.15.93:6820","nonce":16855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6825","nonce":16855},{"type":"v1","addr":"172.21.15.93:6827","nonce":16855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6821","nonce":16855},{"type":"v1","addr":"172.21.15.93:6823","nonce":16855}]},"public_addr":"172.21.15.93:6817/16855","cluster_addr":"172.21.15.93:6820/16855","heartbeat_back_addr":"172.21.15.93:6827/16855","heartbeat_front_addr":"172.21.15.93:6823/16855","state":["exists","up"]},{"osd":5,"uuid":"c5503a49-e4a4-437a-8cb4-43222a82de30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":16854},{"type":"v1","addr":"172.21.15.93:6809","nonce":16854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":16854},{"type":"v1","addr":"172.21.15.93:6811","nonce":16854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":16854},{"type":"v1","addr":"172.21.15.93:6815","nonce":16854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":16854},{"type":"v1","addr":"172.21.15.93:6813","nonce":16854}]},"public_addr":"172.21.15.93:6809/16854","cluster_addr":"172.21.15.93:6811/16854","heartbeat_back_addr":"172.21.15.93:6815/16854","heartbeat_front_addr":"172.21.15.93:6813/16854","state":["exists","up"]},{"osd":6,"uuid":"407a86ed-ec3f-46dd-b55b-97816d373e01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":16856},{"type":"v1","addr":"172.21.15.93:6801","nonce":16856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":16856},{"type":"v1","addr":"172.21.15.93:6803","nonce":16856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":16856},{"type":"v1","addr":"172.21.15.93:6807","nonce":16856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":16856},{"type":"v1","addr":"172.21.15.93:6805","nonce":16856}]},"public_addr":"172.21.15.93:6801/16856","cluster_addr":"172.21.15.93:6803/16856","heartbeat_back_addr":"172.21.15.93:6807/16856","heartbeat_front_addr":"172.21.15.93:6805/16856","state":["exists","up"]},{"osd":7,"uuid":"dacec859-6dea-40cd-933a-765e0db7423e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6819","nonce":16857},{"type":"v1","addr":"172.21.15.93:6822","nonce":16857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":16857},{"type":"v1","addr":"172.21.15.93:6826","nonce":16857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":16857},{"type":"v1","addr":"172.21.15.93:6831","nonce":16857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":16857},{"type":"v1","addr":"172.21.15.93:6829","nonce":16857}]},"public_addr":"172.21.15.93:6822/16857","cluster_addr":"172.21.15.93:6826/16857","heartbeat_back_addr":"172.21.15.93:6831/16857","heartbeat_front_addr":"172.21.15.93:6829/16857","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:36:45.743199+0000","dead_epoch":23},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.676047+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.835637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.222773+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.662450+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.395082+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.967517+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:26.345206+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2021-02-06T00:37:34.512 INFO:tasks.ceph:Scrubbing osd.0 , 2021-02-06T00:37:34.513 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_ceph_fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c/qa/tasks/rados.py", line 265, in thread , run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_git_teuthology_5251f0da07972a7365332a5ff74fa4d328580822/teuthology/orchestra/run.py", line 479, in wait Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2021-02-06T00:38:09.835 INFO:tasks.ceph:Archiving crash dumps... , 2021-02-06T00:38:09.838 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2021-02-05_03:31:02-rados-pacific-distro-basic-smithi/5857765/remote/smithi055/crash Ignoring malparsed backtrace: suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ , next(self.gen) Ignoring malparsed backtrace: ceph version 16.1.0-166-gfc1e361a (fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c) pacific (rc) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe1) [0x7f40b91360ab] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x59c) [0x555e31810c4c] Ignoring malparsed backtrace: 2021-02-06T00:36:37.063 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.841454+0000", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00083392299999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.020388+0000", , 2021-02-06T00:36:37.096 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.1042000000000001e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 104.203411089, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.17825946300000001, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "throttled", , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.846130+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:08.914987+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.193 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:09.079391+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:37.256 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4760000000000001e-06 , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.277 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:39215 1.es0 1:78828434:::smithi05521660-18:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e78)", , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:36:12.056732+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.343994+0000", , 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 1.9063e-05 , 2021-02-06T00:36:37.299 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.367457+0000", , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.2547999999999998e-05 , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 44717 , 2021-02-06T00:36:37.342 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.366 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.824302+0000", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4959999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.843930+0000", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.0020290519999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "MOSDECSubOpWrite(1.fs1 47/18 ECSubWrite(tid=4985, reqid=client.4424.0:17271, at_version=47'1484, trim_to=0'0, roll_forward_to=47'1482))", Ignoring malparsed backtrace: 2021-02-06T00:36:37.827 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.430348+0000", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.3831000000000005e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.850 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.050878+0000", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019642300000000001 Ignoring malparsed backtrace: 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 108.342798449, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61213868100000002, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.442780+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.054998+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_info": { , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client": "client.4424", , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", Ignoring malparsed backtrace: 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:37.977 INFO:teuthology.orchestra.run.smithi093.stdout: "flag_point": "reached pg", , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.105779923 , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: ] Ignoring malparsed backtrace: 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.482205+0000", , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.4496e-05 , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.490276+0000", , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.000131292 , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.060 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:32790 1.as0 1:567b208f:::smithi05521660-19:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e69)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:44.196336+0000", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4800000000000007e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:33:49.401793+0000", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00059431200000000005 Ignoring malparsed backtrace: 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.653920+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.550 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.654352+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.571 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.776178+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.593 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "header_read", Ignoring malparsed backtrace: 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.616 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.637 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19615 1.7s0 1:ea563466:::smithi05521660-14:head [append 0~442368] snapc 0=[] ondisk+write+known_if_redirected e50)", , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:34:52.833245+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.837574+0000", , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4730000000000006e-06 , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.746 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.842891+0000", , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00073171300000000005 , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19619 1.7s0 1:ea563466:::smithi05521660-14:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e50)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.789 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.946122+0000", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 3.0159999999999999e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:38.811 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:19.188380+0000", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.041902981999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:39.227 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.436670+0000", , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61600527599999999 , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 18817 , 2021-02-06T00:36:39.252 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.426192+0000", , 2021-02-06T00:36:39.274 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 5.9059999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.011461+0000", , 2021-02-06T00:36:39.296 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019071299999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:39.319 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867476+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:39.342 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867785+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", , 2021-02-06T00:36:39.365 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.021659+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.386 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.409 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: ] , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } Ignoring malparsed backtrace: 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:39.587 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 2.5142000000000001e-05 , 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.617 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.644 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.6396000000000004e-05 , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.136244222 , 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.672 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: dumped all , 2021-02-06T00:36:39.705 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean , 2021-02-06T00:36:39.705 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2021-02-06T00:37:13.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 3 1 , reweighted osd.3 to 1 (10000) , 2021-02-06T00:37:14.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 3 1 Ignoring malparsed backtrace: 2021-02-06T00:37:33.496 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":125,"fsid":"6c86354a-13de-42b3-8018-2e556a88210c","created":"2021-02-06T00:33:14.600665+0000","modified":"2021-02-06T00:37:26.562048+0000","last_up_change":"2021-02-06T00:36:45.392663+0000","last_in_change":"2021-02-06T00:36:46.404363+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"unique_pool_0","create_time":"2021-02-06T00:33:38.122606+0000","flags":1,"flags_names":"hashpspool","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":true,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"3ab939df-fbe5-45cd-bb5a-abece9e67104","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":22,"up_from":91,"up_thru":0,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6817","nonce":34571},{"type":"v1","addr":"172.21.15.55:6819","nonce":34571}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6821","nonce":34571},{"type":"v1","addr":"172.21.15.55:6823","nonce":34571}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6829","nonce":34571},{"type":"v1","addr":"172.21.15.55:6830","nonce":34571}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6825","nonce":34571},{"type":"v1","addr":"172.21.15.55:6827","nonce":34571}]},"public_addr":"172.21.15.55:6819/34571","cluster_addr":"172.21.15.55:6823/34571","heartbeat_back_addr":"172.21.15.55:6830/34571","heartbeat_front_addr":"172.21.15.55:6827/34571","state":["exists","up"]},{"osd":1,"uuid":"3074b170-47fe-4c88-964b-5b22c9ebe21b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":16890},{"type":"v1","addr":"172.21.15.55:6818","nonce":16890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":16890},{"type":"v1","addr":"172.21.15.55:6822","nonce":16890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":16890},{"type":"v1","addr":"172.21.15.55:6831","nonce":16890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":16890},{"type":"v1","addr":"172.21.15.55:6826","nonce":16890}]},"public_addr":"172.21.15.55:6818/16890","cluster_addr":"172.21.15.55:6822/16890","heartbeat_back_addr":"172.21.15.55:6831/16890","heartbeat_front_addr":"172.21.15.55:6826/16890","state":["exists","up"]},{"osd":2,"uuid":"1f407f8d-c4bd-4c2b-825d-db006892f504","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":16889},{"type":"v1","addr":"172.21.15.55:6809","nonce":16889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":16889},{"type":"v1","addr":"172.21.15.55:6811","nonce":16889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":16889},{"type":"v1","addr":"172.21.15.55:6815","nonce":16889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":16889},{"type":"v1","addr":"172.21.15.55:6813","nonce":16889}]},"public_addr":"172.21.15.55:6809/16889","cluster_addr":"172.21.15.55:6811/16889","heartbeat_back_addr":"172.21.15.55:6815/16889","heartbeat_front_addr":"172.21.15.55:6813/16889","state":["exists","up"]},{"osd":3,"uuid":"2f02446d-eabb-4f93-88a5-0bf475904461","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":16891},{"type":"v1","addr":"172.21.15.55:6801","nonce":16891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":16891},{"type":"v1","addr":"172.21.15.55:6803","nonce":16891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":16891},{"type":"v1","addr":"172.21.15.55:6807","nonce":16891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":16891},{"type":"v1","addr":"172.21.15.55:6805","nonce":16891}]},"public_addr":"172.21.15.55:6801/16891","cluster_addr":"172.21.15.55:6803/16891","heartbeat_back_addr":"172.21.15.55:6807/16891","heartbeat_front_addr":"172.21.15.55:6805/16891","state":["exists","up"]},{"osd":4,"uuid":"17d16afd-4775-4c0d-bfb4-8bdf46058f68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":16855},{"type":"v1","addr":"172.21.15.93:6817","nonce":16855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":16855},{"type":"v1","addr":"172.21.15.93:6820","nonce":16855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6825","nonce":16855},{"type":"v1","addr":"172.21.15.93:6827","nonce":16855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6821","nonce":16855},{"type":"v1","addr":"172.21.15.93:6823","nonce":16855}]},"public_addr":"172.21.15.93:6817/16855","cluster_addr":"172.21.15.93:6820/16855","heartbeat_back_addr":"172.21.15.93:6827/16855","heartbeat_front_addr":"172.21.15.93:6823/16855","state":["exists","up"]},{"osd":5,"uuid":"c5503a49-e4a4-437a-8cb4-43222a82de30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":16854},{"type":"v1","addr":"172.21.15.93:6809","nonce":16854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":16854},{"type":"v1","addr":"172.21.15.93:6811","nonce":16854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":16854},{"type":"v1","addr":"172.21.15.93:6815","nonce":16854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":16854},{"type":"v1","addr":"172.21.15.93:6813","nonce":16854}]},"public_addr":"172.21.15.93:6809/16854","cluster_addr":"172.21.15.93:6811/16854","heartbeat_back_addr":"172.21.15.93:6815/16854","heartbeat_front_addr":"172.21.15.93:6813/16854","state":["exists","up"]},{"osd":6,"uuid":"407a86ed-ec3f-46dd-b55b-97816d373e01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":16856},{"type":"v1","addr":"172.21.15.93:6801","nonce":16856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":16856},{"type":"v1","addr":"172.21.15.93:6803","nonce":16856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":16856},{"type":"v1","addr":"172.21.15.93:6807","nonce":16856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":16856},{"type":"v1","addr":"172.21.15.93:6805","nonce":16856}]},"public_addr":"172.21.15.93:6801/16856","cluster_addr":"172.21.15.93:6803/16856","heartbeat_back_addr":"172.21.15.93:6807/16856","heartbeat_front_addr":"172.21.15.93:6805/16856","state":["exists","up"]},{"osd":7,"uuid":"dacec859-6dea-40cd-933a-765e0db7423e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6819","nonce":16857},{"type":"v1","addr":"172.21.15.93:6822","nonce":16857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":16857},{"type":"v1","addr":"172.21.15.93:6826","nonce":16857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":16857},{"type":"v1","addr":"172.21.15.93:6831","nonce":16857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":16857},{"type":"v1","addr":"172.21.15.93:6829","nonce":16857}]},"public_addr":"172.21.15.93:6822/16857","cluster_addr":"172.21.15.93:6826/16857","heartbeat_back_addr":"172.21.15.93:6831/16857","heartbeat_front_addr":"172.21.15.93:6829/16857","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:36:45.743199+0000","dead_epoch":23},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.676047+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.835637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.222773+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.662450+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.395082+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.967517+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:26.345206+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2021-02-06T00:37:34.512 INFO:tasks.ceph:Scrubbing osd.0 , 2021-02-06T00:37:34.513 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_ceph_fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c/qa/tasks/rados.py", line 265, in thread , run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_git_teuthology_5251f0da07972a7365332a5ff74fa4d328580822/teuthology/orchestra/run.py", line 479, in wait Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2021-02-06T00:38:09.835 INFO:tasks.ceph:Archiving crash dumps... , 2021-02-06T00:38:09.838 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2021-02-05_03:31:02-rados-pacific-distro-basic-smithi/5857765/remote/smithi055/crash Ignoring malparsed backtrace: suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ , next(self.gen) Ignoring malparsed backtrace: ceph version 16.1.0-166-gfc1e361a (fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c) pacific (rc) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe1) [0x7f40b91360ab] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x59c) [0x555e31810c4c] Ignoring malparsed backtrace: 2021-02-06T00:36:37.063 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.841454+0000", , 2021-02-06T00:36:37.064 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00083392299999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.095 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.020388+0000", , 2021-02-06T00:36:37.096 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.1042000000000001e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 104.203411089, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.17825946300000001, , 2021-02-06T00:36:37.124 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "throttled", , 2021-02-06T00:36:37.148 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.846130+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.170 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:08.914987+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.193 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.194 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:09.079391+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.215 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", Ignoring malparsed backtrace: 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.236 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:37.256 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4760000000000001e-06 , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.257 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.277 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:39215 1.es0 1:78828434:::smithi05521660-18:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e78)", , 2021-02-06T00:36:37.278 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:36:12.056732+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.343994+0000", , 2021-02-06T00:36:37.298 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 1.9063e-05 , 2021-02-06T00:36:37.299 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:21.367457+0000", , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.2547999999999998e-05 , 2021-02-06T00:36:37.321 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:37.341 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 44717 , 2021-02-06T00:36:37.342 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:37.366 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.824302+0000", , 2021-02-06T00:36:37.367 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.4959999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:36:36.843930+0000", , 2021-02-06T00:36:37.387 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.0020290519999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.804 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "MOSDECSubOpWrite(1.fs1 47/18 ECSubWrite(tid=4985, reqid=client.4424.0:17271, at_version=47'1484, trim_to=0'0, roll_forward_to=47'1482))", Ignoring malparsed backtrace: 2021-02-06T00:36:37.827 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.430348+0000", , 2021-02-06T00:36:37.828 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 7.3831000000000005e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:37.850 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.050878+0000", , 2021-02-06T00:36:37.851 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019642300000000001 Ignoring malparsed backtrace: 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "age": 108.342798449, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61213868100000002, , 2021-02-06T00:36:37.871 INFO:teuthology.orchestra.run.smithi093.stdout: "type_data": { Ignoring malparsed backtrace: 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:37.892 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.442780+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:37.913 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:50.054998+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_info": { , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client": "client.4424", , 2021-02-06T00:36:37.934 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", Ignoring malparsed backtrace: 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:37.955 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:37.977 INFO:teuthology.orchestra.run.smithi093.stdout: "flag_point": "reached pg", , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:37.978 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.105779923 , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:37.998 INFO:teuthology.orchestra.run.smithi093.stdout: ] Ignoring malparsed backtrace: 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.482205+0000", , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 6.4496e-05 , 2021-02-06T00:36:38.019 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:24.490276+0000", , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.000131292 , 2021-02-06T00:36:38.040 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.060 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.061 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:32790 1.as0 1:567b208f:::smithi05521660-19:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e69)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:44.196336+0000", , 2021-02-06T00:36:38.081 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4800000000000007e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "started", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:33:49.401793+0000", , 2021-02-06T00:36:38.502 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00059431200000000005 Ignoring malparsed backtrace: 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:38.526 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.653920+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.550 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:38.551 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.654352+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.571 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "reached_pg", , 2021-02-06T00:36:38.572 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:33.776178+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.592 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.593 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "header_read", Ignoring malparsed backtrace: 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.615 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.616 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.636 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.637 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.658 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", Ignoring malparsed backtrace: 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:38.680 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19615 1.7s0 1:ea563466:::smithi05521660-14:head [append 0~442368] snapc 0=[] ondisk+write+known_if_redirected e50)", , 2021-02-06T00:36:38.701 INFO:teuthology.orchestra.run.smithi093.stdout: "initiated_at": "2021-02-06T00:34:52.833245+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.837574+0000", , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 9.4730000000000006e-06 , 2021-02-06T00:36:38.724 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.746 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.842891+0000", , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00073171300000000005 , 2021-02-06T00:36:38.747 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:38.769 INFO:teuthology.orchestra.run.smithi093.stdout: "description": "osd_op(client.4424.0:19619 1.7s0 1:ea563466:::smithi05521660-14:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected e50)", Ignoring malparsed backtrace: 2021-02-06T00:36:38.789 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "all_read", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.946122+0000", , 2021-02-06T00:36:38.790 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 3.0159999999999999e-05 Ignoring malparsed backtrace: 2021-02-06T00:36:38.811 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_started", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:35:19.188380+0000", , 2021-02-06T00:36:38.812 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.041902981999999998 Ignoring malparsed backtrace: 2021-02-06T00:36:39.227 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.436670+0000", , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.61600527599999999 , 2021-02-06T00:36:39.228 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "client_addr": "172.21.15.55:0/2031836522", , 2021-02-06T00:36:39.251 INFO:teuthology.orchestra.run.smithi093.stdout: "tid": 18817 , 2021-02-06T00:36:39.252 INFO:teuthology.orchestra.run.smithi093.stdout: }, Ignoring malparsed backtrace: 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", , 2021-02-06T00:36:39.273 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:49.426192+0000", , 2021-02-06T00:36:39.274 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 5.9059999999999999e-06 Ignoring malparsed backtrace: 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "commit_sent", , 2021-02-06T00:36:39.295 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.011461+0000", , 2021-02-06T00:36:39.296 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.00019071299999999999 Ignoring malparsed backtrace: 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.318 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "initiated", , 2021-02-06T00:36:39.319 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867476+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.341 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "queued_for_pg", , 2021-02-06T00:36:39.342 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:52.867785+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.364 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "done", , 2021-02-06T00:36:39.365 INFO:teuthology.orchestra.run.smithi093.stdout: "time": "2021-02-06T00:34:53.021659+0000", Ignoring malparsed backtrace: 2021-02-06T00:36:39.386 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.387 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.408 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.409 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "waiting for rw locks", Ignoring malparsed backtrace: 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: { , 2021-02-06T00:36:39.431 INFO:teuthology.orchestra.run.smithi093.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: ] , 2021-02-06T00:36:39.557 INFO:teuthology.orchestra.run.smithi093.stdout: } Ignoring malparsed backtrace: 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.586 INFO:teuthology.orchestra.run.smithi093.stdout: "events": [ , 2021-02-06T00:36:39.587 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 2.5142000000000001e-05 , 2021-02-06T00:36:39.616 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.617 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.644 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 8.6396000000000004e-05 , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.645 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: "duration": 0.136244222 , 2021-02-06T00:36:39.671 INFO:teuthology.orchestra.run.smithi093.stdout: }, , 2021-02-06T00:36:39.672 INFO:teuthology.orchestra.run.smithi093.stdout: { Ignoring malparsed backtrace: dumped all , 2021-02-06T00:36:39.705 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean , 2021-02-06T00:36:39.705 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2021-02-06T00:37:13.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 3 1 , reweighted osd.3 to 1 (10000) , 2021-02-06T00:37:14.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd primary-affinity 3 1 Ignoring malparsed backtrace: 2021-02-06T00:37:33.496 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":125,"fsid":"6c86354a-13de-42b3-8018-2e556a88210c","created":"2021-02-06T00:33:14.600665+0000","modified":"2021-02-06T00:37:26.562048+0000","last_up_change":"2021-02-06T00:36:45.392663+0000","last_in_change":"2021-02-06T00:36:46.404363+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"unique_pool_0","create_time":"2021-02-06T00:33:38.122606+0000","flags":1,"flags_names":"hashpspool","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":true,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"3ab939df-fbe5-45cd-bb5a-abece9e67104","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":22,"up_from":91,"up_thru":0,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6817","nonce":34571},{"type":"v1","addr":"172.21.15.55:6819","nonce":34571}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6821","nonce":34571},{"type":"v1","addr":"172.21.15.55:6823","nonce":34571}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6829","nonce":34571},{"type":"v1","addr":"172.21.15.55:6830","nonce":34571}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6825","nonce":34571},{"type":"v1","addr":"172.21.15.55:6827","nonce":34571}]},"public_addr":"172.21.15.55:6819/34571","cluster_addr":"172.21.15.55:6823/34571","heartbeat_back_addr":"172.21.15.55:6830/34571","heartbeat_front_addr":"172.21.15.55:6827/34571","state":["exists","up"]},{"osd":1,"uuid":"3074b170-47fe-4c88-964b-5b22c9ebe21b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":16890},{"type":"v1","addr":"172.21.15.55:6818","nonce":16890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":16890},{"type":"v1","addr":"172.21.15.55:6822","nonce":16890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":16890},{"type":"v1","addr":"172.21.15.55:6831","nonce":16890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":16890},{"type":"v1","addr":"172.21.15.55:6826","nonce":16890}]},"public_addr":"172.21.15.55:6818/16890","cluster_addr":"172.21.15.55:6822/16890","heartbeat_back_addr":"172.21.15.55:6831/16890","heartbeat_front_addr":"172.21.15.55:6826/16890","state":["exists","up"]},{"osd":2,"uuid":"1f407f8d-c4bd-4c2b-825d-db006892f504","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":16889},{"type":"v1","addr":"172.21.15.55:6809","nonce":16889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":16889},{"type":"v1","addr":"172.21.15.55:6811","nonce":16889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":16889},{"type":"v1","addr":"172.21.15.55:6815","nonce":16889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":16889},{"type":"v1","addr":"172.21.15.55:6813","nonce":16889}]},"public_addr":"172.21.15.55:6809/16889","cluster_addr":"172.21.15.55:6811/16889","heartbeat_back_addr":"172.21.15.55:6815/16889","heartbeat_front_addr":"172.21.15.55:6813/16889","state":["exists","up"]},{"osd":3,"uuid":"2f02446d-eabb-4f93-88a5-0bf475904461","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":16891},{"type":"v1","addr":"172.21.15.55:6801","nonce":16891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":16891},{"type":"v1","addr":"172.21.15.55:6803","nonce":16891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":16891},{"type":"v1","addr":"172.21.15.55:6807","nonce":16891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":16891},{"type":"v1","addr":"172.21.15.55:6805","nonce":16891}]},"public_addr":"172.21.15.55:6801/16891","cluster_addr":"172.21.15.55:6803/16891","heartbeat_back_addr":"172.21.15.55:6807/16891","heartbeat_front_addr":"172.21.15.55:6805/16891","state":["exists","up"]},{"osd":4,"uuid":"17d16afd-4775-4c0d-bfb4-8bdf46058f68","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":16855},{"type":"v1","addr":"172.21.15.93:6817","nonce":16855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":16855},{"type":"v1","addr":"172.21.15.93:6820","nonce":16855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6825","nonce":16855},{"type":"v1","addr":"172.21.15.93:6827","nonce":16855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6821","nonce":16855},{"type":"v1","addr":"172.21.15.93:6823","nonce":16855}]},"public_addr":"172.21.15.93:6817/16855","cluster_addr":"172.21.15.93:6820/16855","heartbeat_back_addr":"172.21.15.93:6827/16855","heartbeat_front_addr":"172.21.15.93:6823/16855","state":["exists","up"]},{"osd":5,"uuid":"c5503a49-e4a4-437a-8cb4-43222a82de30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":16854},{"type":"v1","addr":"172.21.15.93:6809","nonce":16854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":16854},{"type":"v1","addr":"172.21.15.93:6811","nonce":16854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":16854},{"type":"v1","addr":"172.21.15.93:6815","nonce":16854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":16854},{"type":"v1","addr":"172.21.15.93:6813","nonce":16854}]},"public_addr":"172.21.15.93:6809/16854","cluster_addr":"172.21.15.93:6811/16854","heartbeat_back_addr":"172.21.15.93:6815/16854","heartbeat_front_addr":"172.21.15.93:6813/16854","state":["exists","up"]},{"osd":6,"uuid":"407a86ed-ec3f-46dd-b55b-97816d373e01","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":16856},{"type":"v1","addr":"172.21.15.93:6801","nonce":16856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":16856},{"type":"v1","addr":"172.21.15.93:6803","nonce":16856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":16856},{"type":"v1","addr":"172.21.15.93:6807","nonce":16856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":16856},{"type":"v1","addr":"172.21.15.93:6805","nonce":16856}]},"public_addr":"172.21.15.93:6801/16856","cluster_addr":"172.21.15.93:6803/16856","heartbeat_back_addr":"172.21.15.93:6807/16856","heartbeat_front_addr":"172.21.15.93:6805/16856","state":["exists","up"]},{"osd":7,"uuid":"dacec859-6dea-40cd-933a-765e0db7423e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6819","nonce":16857},{"type":"v1","addr":"172.21.15.93:6822","nonce":16857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":16857},{"type":"v1","addr":"172.21.15.93:6826","nonce":16857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":16857},{"type":"v1","addr":"172.21.15.93:6831","nonce":16857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":16857},{"type":"v1","addr":"172.21.15.93:6829","nonce":16857}]},"public_addr":"172.21.15.93:6822/16857","cluster_addr":"172.21.15.93:6826/16857","heartbeat_back_addr":"172.21.15.93:6831/16857","heartbeat_front_addr":"172.21.15.93:6829/16857","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:36:45.743199+0000","dead_epoch":23},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.676047+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.835637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:25.222773+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.662450+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.395082+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:24.967517+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-06T00:37:26.345206+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2021-02-06T00:37:34.512 INFO:tasks.ceph:Scrubbing osd.0 , 2021-02-06T00:37:34.513 DEBUG:teuthology.orchestra.run.smithi055:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_ceph_fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c/qa/tasks/rados.py", line 265, in thread , run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_git_teuthology_5251f0da07972a7365332a5ff74fa4d328580822/teuthology/orchestra/run.py", line 479, in wait Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2021-02-06T00:38:09.835 INFO:tasks.ceph:Archiving crash dumps... , 2021-02-06T00:38:09.838 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2021-02-05_03:31:02-rados-pacific-distro-basic-smithi/5857765/remote/smithi055/crash Ignoring malparsed backtrace: suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ , next(self.gen) Ignoring malparsed backtrace: ceph version 16.1.0-166-gfc1e361a (fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c) pacific (rc) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f2cd383e15a] , 2: (ReadOp::_finish(TestOp::CallbackInfo*)+0x1306) [0x555580f09306] Ignoring malparsed backtrace: 2021-02-06T00:11:28.055 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-02-06T00:11:28.328 INFO:teuthology.orchestra.run.smithi033.stdout: , dumped all Ignoring malparsed backtrace: 2021-02-06T00:13:10.377 DEBUG:tasks.ceph.osd.2:waiting for process to exit , 2021-02-06T00:13:10.378 INFO:teuthology.orchestra.run:waiting for 300 , 2021-02-06T00:13:10.479 INFO:tasks.ceph.osd.2:Stopped Ignoring malparsed backtrace: 2021-02-06T00:17:45.151 INFO:teuthology.orchestra.run.smithi033.stdout:^* hv02.front.sepia.ceph.com 2 6 377 55 -217us[ -216us] +/- 33ms , 2021-02-06T00:17:45.151 INFO:teuthology.orchestra.run.smithi033.stdout:^- hv03.front.sepia.ceph.com 3 6 377 58 -1031us[-1030us] +/- 65ms , 2021-02-06T00:17:45.152 INFO:teuthology.orchestra.run.smithi033.stdout:^+ hv04.front.sepia.ceph.com 3 6 377 57 +382us[ +383us] +/- 57ms Ignoring malparsed backtrace: ceph version 16.1.0-166-gfc1e361a (fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c) pacific (rc) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f2cd383e15a] , 2: (ReadOp::_finish(TestOp::CallbackInfo*)+0x1306) [0x555580f09306] Ignoring malparsed backtrace: 2021-02-06T00:11:28.055 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-02-06T00:11:28.328 INFO:teuthology.orchestra.run.smithi033.stdout: , dumped all Ignoring malparsed backtrace: 2021-02-06T00:13:10.377 DEBUG:tasks.ceph.osd.2:waiting for process to exit , 2021-02-06T00:13:10.378 INFO:teuthology.orchestra.run:waiting for 300 , 2021-02-06T00:13:10.479 INFO:tasks.ceph.osd.2:Stopped Ignoring malparsed backtrace: 2021-02-06T00:17:45.151 INFO:teuthology.orchestra.run.smithi033.stdout:^* hv02.front.sepia.ceph.com 2 6 377 55 -217us[ -216us] +/- 33ms , 2021-02-06T00:17:45.151 INFO:teuthology.orchestra.run.smithi033.stdout:^- hv03.front.sepia.ceph.com 3 6 377 58 -1031us[-1030us] +/- 65ms , 2021-02-06T00:17:45.152 INFO:teuthology.orchestra.run.smithi033.stdout:^+ hv04.front.sepia.ceph.com 3 6 377 57 +382us[ +383us] +/- 57ms Found 8 distinct failure reasons Dead: b'2021-02-06T12:48:45.405 DEBUG:teuthology.exit:Finished running handlers' 2 jobs: ['5857769', '5857281'] suites intersection: ['2-recovery-overrides/{default}', 'ceph', 'clusters/{fixed-2', 'openstack}', 'rados', 'thrashosds-health', 'workloads/radosbench-high-concurrency}'] suites union: ['1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', 'backoff/normal', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashers/default', 'thrashosds-health', 'workloads/radosbench-high-concurrency}'] Failure: Test failure: test_all (tasks.mgr.dashboard.test_rgw.RgwBucketTest) 1 jobs: ['5857389'] suites: ['debug/mgr', 'mon_election/connectivity', 'objectstore/bluestore-comp-zlib', 'rados/dashboard/{clusters/{2-node-mgr}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/dashboard}'] Failure: Command crashed: 'CEPH_CLIENT_ID=2 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 4000 --objects 500 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 50 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --op copy_from 50 --op cache_flush 50 --op cache_try_flush 50 --op cache_evict 50 --op write_excl 50 --pool base' 2 jobs: ['5857229', '5857699'] suites intersection: ['mon_election/classic', 'rados'] suites union: ['1-install/luminous-v1only', 'backoff/peering_and_degraded', 'ceph', 'clusters/{openstack', 'd-balancer/crush-compat', 'distro$/{ubuntu_18.04}', 'mon_election/classic', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr/async-v1only', 'objectstore/bluestore-comp-zstd', 'rados', 'rados/cephadm/thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size', 'rados/singleton/{all/thrash_cache_writeback_proxy_none', 'supported-random-distro$/{rhel_8}}', 'thrashers/mapgap', 'thrashosds-health', 'three-plus-one}', 'workloads/cache-snaps}'] Failure: Test failure: test_delete (tasks.mgr.dashboard.test_rgw.RgwUserCapabilityTest) 1 jobs: ['5857669'] suites: ['debug/mgr', 'mon_election/classic', 'objectstore/bluestore-comp-lz4', 'rados/dashboard/{clusters/{2-node-mgr}', 'supported-random-distro$/{rhel_8}', 'tasks/dashboard}'] Failure: Command failed (workunit test rados/test.sh) on smithi154 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=fc1e361ac9e05b37d8f9c5ac0b433ca305ec485c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test.sh' 1 jobs: ['5857245'] suites: ['clusters/{fixed-2', 'mon_election/classic', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'rados', 'rados/basic/{ceph', 'supported-random-distro$/{centos_8}', 'tasks/rados_api_tests}'] Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=nautilus 2 jobs: ['5857630', '5857265'] suites intersection: ['1-install/nautilus', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'bluestore-bitmap', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rbd-import-export', 'rbd_api}', 'readwrite', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'thrashosds-health', 'ubuntu_latest}'] suites union: ['1-install/nautilus', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'bluestore-bitmap', 'mon_election/classic', 'mon_election/connectivity', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rbd-import-export', 'rbd_api}', 'readwrite', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'thrashosds-health', 'ubuntu_latest}'] Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --no-omap --ec-pool --max-ops 400000 --objects 20 --max-in-flight 8 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 0 --op write 0 --op delete 20 --op append 5 --op write_excl 0 --op append_excl 5 --pool unique_pool_0' 1 jobs: ['5857765'] suites: ['clusters/{fixed-2', 'fast/fast', 'mon_election/classic', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'openstack}', 'rados', 'rados/thrash-erasure-code/{ceph', 'recovery-overrides/{more-async-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/minsize_recovery', 'thrashosds-health', 'workloads/ec-small-objects-many-deletes}'] Failure: "2021-02-06T01:13:41.621299+0000 mon.a (mon.0) 20 : cluster [WRN] Health check failed: 3/21 mons down, quorum a,b,c,d,e,f,g,h,i,j,k,l,o,p,r,s,t,u (MON_DOWN)" in cluster log 1 jobs: ['5857837'] suites: ['mon_election/classic', 'msgr-failures/few', 'msgr/async', 'no_pools', 'objectstore/bluestore-comp-zlib', 'rados', 'rados/multimon/{clusters/21', 'supported-random-distro$/{centos_8}', 'tasks/mon_clock_no_skews}']